From 03fca61fcbdc0cde2b97bcd0b22e0efe4709d93a Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 20 Mar 2020 14:04:33 +0100 Subject: [PATCH] [DOCS] add docs for async search (#53675) Relates to #49091 Co-Authored-By: James Rodewig --- .../doc/RestTestsFromSnippetsTask.groovy | 3 - docs/reference/api-conventions.asciidoc | 6 +- docs/reference/async-search.asciidoc | 22 + docs/reference/index.asciidoc | 2 + .../reference/indices/rollover-index.asciidoc | 4 +- docs/reference/redirects.asciidoc | 355 +- docs/reference/search.asciidoc | 2 + docs/reference/search/async-search.asciidoc | 214 + t | 509337 +++++++++++++++ .../api/async_search.delete.json | 2 +- .../rest-api-spec/api/async_search.get.json | 2 +- .../api/async_search.submit.json | 18 +- 12 files changed, 509768 insertions(+), 199 deletions(-) create mode 100644 docs/reference/async-search.asciidoc create mode 100644 docs/reference/search/async-search.asciidoc create mode 100644 t diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy index 393c93c54e8..021646724ae 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy @@ -307,10 +307,7 @@ class RestTestsFromSnippetsTask extends SnippetsTask { if (path == null) { path = '' // Catch requests to the root... } else { - // Escape some characters that are also escaped by sense path = path.replace('<', '%3C').replace('>', '%3E') - path = path.replace('{', '%7B').replace('}', '%7D') - path = path.replace('|', '%7C') } current.println(" - do:") if (catchPart != null) { diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 43909fdb7e5..7249a597842 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -87,7 +87,7 @@ GET /%3Clogstash-%7Bnow%2Fd%7D%3E/_search } ---------------------------------------------------------------------- // TEST[s/^/PUT logstash-2016.09.20\n/] -// TEST[s/now/2016.09.20||/] +// TEST[s/now/2016.09.20%7C%7C/] [NOTE] .Percent encoding of date math characters @@ -141,7 +141,7 @@ GET /%3Clogstash-%7Bnow%2Fd-2d%7D%3E%2C%3Clogstash-%7Bnow%2Fd-1d%7D%3E%2C%3Clogs } ---------------------------------------------------------------------- // TEST[s/^/PUT logstash-2016.09.20\nPUT logstash-2016.09.19\nPUT logstash-2016.09.18\n/] -// TEST[s/now/2016.09.20||/] +// TEST[s/now/2016.09.20%7C%7C/] [[common-options]] === Common options @@ -367,7 +367,7 @@ GET /_search?filter_path=hits.hits._source&_source=title&sort=rating:desc [float] ==== Flat Settings -The `flat_settings` flag affects rendering of the lists of settings. When the +The `flat_settings` flag affects rendering of the lists of settings. When the `flat_settings` flag is `true`, settings are returned in a flat format: [source,console] diff --git a/docs/reference/async-search.asciidoc b/docs/reference/async-search.asciidoc new file mode 100644 index 00000000000..d51e017b0f9 --- /dev/null +++ b/docs/reference/async-search.asciidoc @@ -0,0 +1,22 @@ +[role="xpack"] +[testenv="basic"] +[[async-search-intro]] +== Long-running searches + +{es} generally allows you to quickly search across big amounts of data. There are +situations where a search executes on many many shards, possibly against +<> and spanning multiple +<>, for which +results are not expected to be returned in milliseconds. When you need to +execute long-running searches, synchronously +waiting for its results to be returned is not ideal. Instead, Async search lets +you submit a search request that gets executed _asynchronously_, +monitor the progress of the request, and retrieve results at a later stage. +You can also retrieve partial results as they become available but +before the search has completed. + +You can submit an async search request using the <> API. The <> API allows you to +monitor the progress of an async search request and retrieve its results. An +ongoing async search can be deleted through the <> API. diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 6ecb6a7b856..7b9edda4937 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -30,6 +30,8 @@ include::query-dsl.asciidoc[] include::modules/cross-cluster-search.asciidoc[] +include::async-search.asciidoc[] + include::scripting.asciidoc[] include::mapping.asciidoc[] diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 46a1ae10f59..7e5da9bb40b 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -302,7 +302,7 @@ POST /logs_write/_rollover <2> } } -------------------------------------------------- -// TEST[s/now/2016.10.31||/] +// TEST[s/now/2016.10.31%7C%7C/] <1> Creates an index named with today's date (e.g.) `logs-2016.10.31-1` <2> Rolls over to a new index with today's date, e.g. `logs-2016.10.31-000002` if run immediately, or `logs-2016.11.01-000002` if run after 24 hours @@ -341,7 +341,7 @@ over indices created in the last three days, you could do the following: GET /%3Clogs-%7Bnow%2Fd%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-1d%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-2d%7D-*%3E/_search -------------------------------------------------- // TEST[continued] -// TEST[s/now/2016.10.31||/] +// TEST[s/now/2016.10.31%7C%7C/] [[rollover-index-api-dry-run-ex]] diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 345b93d33ee..6f939d0dab9 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -25,227 +25,227 @@ index that make warmers not necessary anymore. See <>. -[role="exclude",id="xpack-api"] -=== X-Pack APIs +[role="exclude",id="xpack-api"] +=== X-Pack APIs -{es} {xpack} APIs are now documented in <>. +{es} {xpack} APIs are now documented in <>. -[role="exclude",id="ml-calendar-resource"]] -=== Calendar resources +[role="exclude",id="ml-calendar-resource"]] +=== Calendar resources -See <> and -{ml-docs}/ml-calendars.html[Calendars and scheduled events]. +See <> and +{ml-docs}/ml-calendars.html[Calendars and scheduled events]. -[role="exclude",id="ml-filter-resource"] -=== Filter resources +[role="exclude",id="ml-filter-resource"] +=== Filter resources -See <> and -{ml-docs}/ml-rules.html[Machine learning custom rules]. +See <> and +{ml-docs}/ml-rules.html[Machine learning custom rules]. -[role="exclude",id="ml-event-resource"] -=== Scheduled event resources +[role="exclude",id="ml-event-resource"] +=== Scheduled event resources -See <> and -{ml-docs}/ml-calendars.html[Calendars and scheduled events]. +See <> and +{ml-docs}/ml-calendars.html[Calendars and scheduled events]. -[role="exclude",id="index-apis"] -=== Index APIs -{es} index APIs are now documented in <>. +[role="exclude",id="index-apis"] +=== Index APIs +{es} index APIs are now documented in <>. -[role="exclude",id="search-request-docvalue-fields"] -=== Doc value fields parameter for request body search API -See <>. +[role="exclude",id="search-request-docvalue-fields"] +=== Doc value fields parameter for request body search API +See <>. -[role="exclude",id="search-request-explain"] -=== Explain parameter for request body search API -See <>. +[role="exclude",id="search-request-explain"] +=== Explain parameter for request body search API +See <>. -[role="exclude",id="search-request-collapse"] -=== Collapse parameter for request body search API -See <>. +[role="exclude",id="search-request-collapse"] +=== Collapse parameter for request body search API +See <>. -[role="exclude",id="search-request-from-size"] -=== From and size parameters for request body search API -See <>. +[role="exclude",id="search-request-from-size"] +=== From and size parameters for request body search API +See <>. -[role="exclude",id="search-request-highlighting"] -=== Highlight parameter for request body search API -See <>. +[role="exclude",id="search-request-highlighting"] +=== Highlight parameter for request body search API +See <>. -[role="exclude",id="search-request-index-boost"] -=== Index boost parameter for request body search API -See <>. +[role="exclude",id="search-request-index-boost"] +=== Index boost parameter for request body search API +See <>. -[role="exclude",id="search-request-inner-hits"] -=== Inner hits parameter for request body search API -See <>. +[role="exclude",id="search-request-inner-hits"] +=== Inner hits parameter for request body search API +See <>. -[role="exclude",id="search-request-min-score"] -=== Minimum score parameter for request body search API -See <>. +[role="exclude",id="search-request-min-score"] +=== Minimum score parameter for request body search API +See <>. -[role="exclude",id="search-request-named-queries-and-filters"] -=== Named query parameter for request body search API -See <>. +[role="exclude",id="search-request-named-queries-and-filters"] +=== Named query parameter for request body search API +See <>. -[role="exclude",id="search-request-post-filter"] -=== Post filter parameter for request body search API -See <>. +[role="exclude",id="search-request-post-filter"] +=== Post filter parameter for request body search API +See <>. -[role="exclude",id="search-request-preference"] -=== Preference parameter for request body search API -See <>. +[role="exclude",id="search-request-preference"] +=== Preference parameter for request body search API +See <>. -[role="exclude",id="search-request-query"] -=== Query parameter for request body search API -See <>. +[role="exclude",id="search-request-query"] +=== Query parameter for request body search API +See <>. -[role="exclude",id="search-request-rescore"] -=== Rescoring parameter for request body search API -See <>. +[role="exclude",id="search-request-rescore"] +=== Rescoring parameter for request body search API +See <>. -[role="exclude",id="search-request-script-fields"] -=== Script fields parameter for request body search API -See <>. +[role="exclude",id="search-request-script-fields"] +=== Script fields parameter for request body search API +See <>. -[role="exclude",id="search-request-scroll"] -=== Scroll parameter for request body search API -See <>. +[role="exclude",id="search-request-scroll"] +=== Scroll parameter for request body search API +See <>. -[role="exclude",id="search-request-search-after"] -=== Search after parameter for request body search API -See <>. +[role="exclude",id="search-request-search-after"] +=== Search after parameter for request body search API +See <>. -[role="exclude",id="search-request-search-type"] -=== Search type parameter for request body search API -See <>. +[role="exclude",id="search-request-search-type"] +=== Search type parameter for request body search API +See <>. -[role="exclude",id="search-request-seq-no-primary-term"] -=== Sequence numbers and primary terms parameter for request body search API -See <>. +[role="exclude",id="search-request-seq-no-primary-term"] +=== Sequence numbers and primary terms parameter for request body search API +See <>. -[role="exclude",id="search-request-sort"] -=== Sort parameter for request body search API -See <>. +[role="exclude",id="search-request-sort"] +=== Sort parameter for request body search API +See <>. -[role="exclude",id="search-request-source-filtering"] -=== Source filtering parameter for request body search API -See <>. +[role="exclude",id="search-request-source-filtering"] +=== Source filtering parameter for request body search API +See <>. -[role="exclude",id="search-request-stored-fields"] -=== Stored fields parameter for request body search API -See <>. +[role="exclude",id="search-request-stored-fields"] +=== Stored fields parameter for request body search API +See <>. -[role="exclude",id="search-request-track-total-hits"] -=== Track total hits parameter for request body search API -See <>. +[role="exclude",id="search-request-track-total-hits"] +=== Track total hits parameter for request body search API +See <>. -[role="exclude",id="search-request-version"] -=== Version parameter for request body search API -See <>. +[role="exclude",id="search-request-version"] +=== Version parameter for request body search API +See <>. -[role="exclude",id="search-suggesters-term"] -=== Term suggester -See <>. +[role="exclude",id="search-suggesters-term"] +=== Term suggester +See <>. -[role="exclude",id="search-suggesters-phrase"] -=== Phrase suggester -See <>. +[role="exclude",id="search-suggesters-phrase"] +=== Phrase suggester +See <>. -[role="exclude",id="search-suggesters-completion"] -=== Completion suggester -See <>. +[role="exclude",id="search-suggesters-completion"] +=== Completion suggester +See <>. -[role="exclude",id="suggester-context"] -=== Context suggester -See <>. +[role="exclude",id="suggester-context"] +=== Context suggester +See <>. -[role="exclude",id="returning-suggesters-type"] -=== Return suggester type -See <>. +[role="exclude",id="returning-suggesters-type"] +=== Return suggester type +See <>. -[role="exclude",id="search-profile-queries"] -=== Profiling queries -See <>. +[role="exclude",id="search-profile-queries"] +=== Profiling queries +See <>. -[role="exclude",id="search-profile-aggregations"] -=== Profiling aggregations -See <>. +[role="exclude",id="search-profile-aggregations"] +=== Profiling aggregations +See <>. -[role="exclude",id="search-profile-considerations"] -=== Profiling considerations -See <>. +[role="exclude",id="search-profile-considerations"] +=== Profiling considerations +See <>. -[role="exclude",id="_explain_analyze"] -=== Explain analyze API -See <>. +[role="exclude",id="_explain_analyze"] +=== Explain analyze API +See <>. -[role="exclude",id="indices-synced-flush"] -=== Synced flush API -See <>. +[role="exclude",id="indices-synced-flush"] +=== Synced flush API +See <>. -[role="exclude",id="_repositories"] -=== Snapshot repositories -See <>. +[role="exclude",id="_repositories"] +=== Snapshot repositories +See <>. -[role="exclude",id="_snapshot"] -=== Snapshot -See <>. +[role="exclude",id="_snapshot"] +=== Snapshot +See <>. -[role="exclude",id="getting-started-explore"] -=== Exploring your cluster -See <>. +[role="exclude",id="getting-started-explore"] +=== Exploring your cluster +See <>. -[role="exclude",id="getting-started-cluster-health"] -=== Cluster health -See <>. +[role="exclude",id="getting-started-cluster-health"] +=== Cluster health +See <>. -[role="exclude", id="getting-started-list-indices"] -=== List all indices -See <>. +[role="exclude", id="getting-started-list-indices"] +=== List all indices +See <>. -[role="exclude", id="getting-started-create-index"] -=== Create an index -See <>. +[role="exclude", id="getting-started-create-index"] +=== Create an index +See <>. -[role="exclude", id="getting-started-query-document"] -=== Index and query a document -See <>. +[role="exclude", id="getting-started-query-document"] +=== Index and query a document +See <>. -[role="exclude", id="getting-started-delete-index"] -=== Delete an index -See <>. +[role="exclude", id="getting-started-delete-index"] +=== Delete an index +See <>. -[role="exclude", id="getting-started-modify-data"] -== Modifying your data -See <>. +[role="exclude", id="getting-started-modify-data"] +== Modifying your data +See <>. -[role="exclude", id="indexing-replacing-documents"] -=== Indexing/replacing documents -See <>. +[role="exclude", id="indexing-replacing-documents"] +=== Indexing/replacing documents +See <>. -[role="exclude", id="getting-started-explore-data"] -=== Exploring your data -See <>. +[role="exclude", id="getting-started-explore-data"] +=== Exploring your data +See <>. -[role="exclude", id="getting-started-search-API"] -=== Search API -See <>. +[role="exclude", id="getting-started-search-API"] +=== Search API +See <>. -[role="exclude", id="getting-started-conclusion"] -=== Conclusion -See <>. +[role="exclude", id="getting-started-conclusion"] +=== Conclusion +See <>. -[role="exclude",id="ccs-reduction"] -=== {ccs-cap} reduction -See <>. +[role="exclude",id="ccs-reduction"] +=== {ccs-cap} reduction +See <>. -[role="exclude",id="administer-elasticsearch"] -=== Administering {es} -See <>. +[role="exclude",id="administer-elasticsearch"] +=== Administering {es} +See <>. -[role="exclude",id="slm-api"] -=== Snapshot lifecycle management API +[role="exclude",id="slm-api"] +=== Snapshot lifecycle management API See <>. [role="exclude",id="delete-data-frame-transform"] @@ -347,7 +347,7 @@ See <>. [role="exclude",id="native-settings"] ==== Native realm settings -See <>. +See <>. [role="exclude",id="configuring-saml-realm"] === Configuring a SAML realm @@ -357,27 +357,27 @@ See <>. [role="exclude",id="saml-settings"] ==== SAML realm settings -See <>. +See <>. [role="exclude",id="_saml_realm_signing_settings"] ==== SAML realm signing settings -See <>. +See <>. [role="exclude",id="_saml_realm_encryption_settings"] ==== SAML realm encryption settings -See <>. +See <>. [role="exclude",id="_saml_realm_ssl_settings"] ==== SAML realm SSL settings -See <>. +See <>. [role="exclude",id="configuring-file-realm"] === Configuring a file realm -See <>. +See <>. [role="exclude",id="ldap-user-search"] === User search mode and user DN templates mode @@ -397,7 +397,7 @@ See <>. [role="exclude",id="ldap-ssl"] === Setting up SSL between Elasticsearch and LDAP -See <>. +See <>. [role="exclude",id="configuring-kerberos-realm"] === Configuring a Kerberos realm @@ -438,7 +438,7 @@ See <>. [role="exclude",id="mapping-roles-ad"] === Mapping Active Directory users and groups to roles -See <>. +See <>. [role="exclude",id="how-security-works"] === How security works @@ -471,9 +471,9 @@ See the details in This page was deleted. [[ml-datafeed-chunking-config]] -See the details in <>, <>, +See the details in <>, <>, [[ml-datafeed-delayed-data-check-config]] -<>, +<>, [[ml-datafeed-counts]] <>. @@ -508,7 +508,7 @@ See <>. [role="exclude",id="ml-dfa-analysis-objects"] === Analysis configuration objects -This page was deleted. +This page was deleted. See <>. [role="exclude",id="data-frames-settings"] @@ -603,11 +603,6 @@ See <>. See <> and <>. -[role="exclude",id="async-search"] -=== Asynchronous search - -coming::[7.x] - [role="exclude",id="data-streams"] === Data stream APIs diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index e8a8a42da6a..a33807cf698 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -152,6 +152,8 @@ high). This default value is `5`. include::search/search.asciidoc[] +include::search/async-search.asciidoc[] + include::search/uri-request.asciidoc[] include::search/request-body.asciidoc[] diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc new file mode 100644 index 00000000000..e58455146fb --- /dev/null +++ b/docs/reference/search/async-search.asciidoc @@ -0,0 +1,214 @@ +[role="xpack"] +[testenv="basic"] +[[async-search]] +=== Async search + +The async search API let you asynchronously execute a +search request, monitor its progress, and retrieve partial results +as they become available. + +[[submit-async-search]] +==== Submit async search API + +Executes a search request asynchronously. It accepts the same +parameters and request body as the <>. + +[source,console,id=submit-async-search-date-histogram-example] +-------------------------------------------------- +POST /sales*/_async_search?size=0 +{ + "sort" : [ + { "date" : {"order" : "asc"} } + ], + "aggs" : { + "sale_date" : { + "date_histogram" : { + "field" : "date", + "calendar_interval": "1d" + } + } + } +} +-------------------------------------------------- +// TEST[setup:sales] +// TEST[s/size=0/size=0&wait_for_completion=0/] + +The response contains an identifier of the search being executed. +You can use this ID to later retrieve the search's final results. +The currently available search +results are returned as part of the <> object. + +[source,console-result] +-------------------------------------------------- +{ + "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=", <1> + "version" : 0, + "is_partial" : true, <2> + "is_running" : true, <3> + "start_time_in_millis" : 1583945890986, + "expiration_time_in_millis" : 1584377890986, + "response" : { + "took" : 1122, + "timed_out" : false, + "num_reduce_phases" : 0, + "_shards" : { + "total" : 562, <4> + "successful" : 3, <5> + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 157483, <6> + "relation" : "gte" + }, + "max_score" : null, + "hits" : [ ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/$body.id/] +// TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/] +// TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/] +// TESTRESPONSE[s/"took" : 1122/"took": $body.response.took/] +// TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/] +// TESTRESPONSE[s/"successful" : 3/"successful": $body.response._shards.successful/] +// TESTRESPONSE[s/"value" : 157483/"value": $body.response.hits.total.value/] + +<1> Identifier of the async search that can be used to monitor its progress, retrieve its results, and/or delete it. +<2> Whether the returned search results are partial or final +<3> Whether the search is still being executed or it has completed +<4> How many shards the search will be executed on, overall +<5> How many shards have successfully completed the search +<6> How many documents are currently matching the query, which belong to the shards that have already completed the search + +It is possible to block and wait until the search is completed up to a certain +timeout by providing the `wait_for_completion` parameter, which defaults to +`1` second. + +You can also specify how long the async search needs to be +available through the `keep_alive` parameter, which defaults to `5d` (five days). +Ongoing async searches and any saved search results are deleted after this +period. + +NOTE: When results are sorted by a numeric field, shards get sorted based on +minimum and maximum value that they hold for that field, hence partial +results become available following the sort criteria that was requested. + +The submit async search API supports the same <> +as the search API, though some have different default values: + +* `batched_reduce_size` defaults to `5`: this affects how often partial results +become available, which happens whenever shard results are reduced. A partial +reduction is performed every time the coordinating node has received a certain +number of new shard responses (`5` by default). +* `request_cache` defaults to `true` +* `pre_filter_shard_size` defaults to `1`: this is to enforce the execution of +a pre-filter roundtrip to retrieve statistics from each shard so that the ones +that surely don't hold any document matching the query get skipped. +* `ccs_minimize_roundtrips` defaults to `false`, which is also the only +supported value + +WARNING: Async search does not support <> +nor search requests that only include the <>. +{ccs} is supported only with <> +set to `false`. + +[[get-async-search]] +==== Get async search + +The get async search API retrieves the results of a previously submitted +async search request given its id. If the {es} {security-features} are enabled. +the access to the results of a specific async search is restricted to the user +that submitted it in the first place. + +[source,console,id=get-async-search-date-histogram-example] +-------------------------------------------------- +GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc= +-------------------------------------------------- +// TEST[continued s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/\${body.id}/] + +[source,console-result] +-------------------------------------------------- +{ + "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=", + "version" : 2, <1> + "is_partial" : true, <2> + "is_running" : true, <3> + "start_time_in_millis" : 1583945890986, + "expiration_time_in_millis" : 1584377890986, <4> + "response" : { + "took" : 12144, + "timed_out" : false, + "num_reduce_phases" : 38, + "_shards" : { + "total" : 562, + "successful" : 188, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 456433, + "relation" : "eq" + }, + "max_score" : null, + "hits" : [ ] + }, + "aggregations" : { <5> + "sale_date" : { + "buckets" : [] + } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/$body.id/] +// TESTRESPONSE[s/"is_partial" : true/"is_partial" : false/] +// TESTRESPONSE[s/"is_running" : true/"is_running" : false/] +// TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/] +// TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/] +// TESTRESPONSE[s/"took" : 12144/"took": $body.response.took/] +// TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/] +// TESTRESPONSE[s/"successful" : 188/"successful": $body.response._shards.successful/] +// TESTRESPONSE[s/"value" : 456433/"value": $body.response.hits.total.value/] +// TESTRESPONSE[s/"buckets" : \[\]/"buckets": $body.response.aggregations.sale_date.buckets/] +// TESTRESPONSE[s/"num_reduce_phases" : 38,//] + +<1> The returned `version` is useful to identify whether the response contains +additional results compared to previously obtained responses. If the version +stays the same, no new results have become available, otherwise a higher version +number indicates that more shards have completed their execution of the query +and their partial results are also included in the response. +<2> Whether the returned search results are partial or final +<3> Whether the search is still being executed or it has completed +<4> When the async search will expire +<5> Partial aggregations results, coming from the shards that have already +completed the execution of the query. + +The `wait_for_completion` parameter, which defaults to `1`, can also be provided +when calling the Get Async Search API, in order to wait for the search to be +completed up until the provided timeout. Final results will be returned if +available before the timeout expires, otherwise the currently available results +will be returned once the timeout expires. + +The `keep_alive` parameter specifies how long the async search should be +available in the cluster. When not specified, the `keep_alive` set with the +corresponding submit async request will be used. Otherwise, it is possible to +override such value and extend the validity of the request. When this period +expires, the search, if still running, is cancelled. If the search is +completed, its saved results are deleted. + +[[delete-async-search]] +==== Delete async search + +You can use the delete async search API to manually delete an async search +by ID. If the search is still running, the search request will be cancelled. +Otherwise, the saved search results are deleted. + +[source,console,id=delete-async-search-date-histogram-example] +-------------------------------------------------- +DELETE /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc= +-------------------------------------------------- +// TEST[continued s/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=/\${body.id}/] diff --git a/t b/t new file mode 100644 index 00000000000..ebe0700dab8 --- /dev/null +++ b/t @@ -0,0 +1,509337 @@ +commit 2b087de29f7d2eb6f4eaa919de9bc20cfed19eb8 (HEAD) +Author: Luca Cavanna +Date: Fri Mar 20 14:45:10 2020 +0100 + + fix + +commit 6c57681446156815275a0deda1433d62ce20a2ca (es/7.x) +Author: James Rodewig +Date: Fri Mar 20 09:40:51 2020 -0400 + + [DOCS] Add redirects for missing data stream API docs (#53866) + + The following API spec files contain a link to a not-yet-created + docs page for the data stream APIs: + + * [indices.create_data_stream.json][0] + * [indices.delete_data_stream.jsonn][1] + * [indices.get_data_streams.json][2] + + The Elaticsearch-JS client uses these spec files to create their docs. + This created a broken link in the Elaticsearch-JS, which has broken + the docs build. + + This PR adds a temporary redirect for the docs page. This redirect + should be removed when the actual API docs are added. + + [0]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json + [1]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json + [2]: https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json + + Relates to #53558. + + CC @martijnvg + +commit 8eacb153df398e87d95cf6516e86d75831f58a99 +Author: Christoph Büscher +Date: Fri Mar 20 13:15:58 2020 +0100 + + Add async_search.submit to HLRC #53592 (#53852) + + This commit adds a new AsyncSearchClient to the High Level Rest Client which + initially supporst the submitAsyncSearch in its blocking and non-blocking + flavour. Also adding client side request and response objects and parsing code + to parse the xContent output of the client side AsyncSearchResponse together + with parsing roundtrip tests and a simple roundtrip integration test. + + Relates to #49091 + Backport of #53592 + +commit 10a2a95b2520f31b9730e1bbf049d80135161326 +Author: Alan Woodward +Date: Fri Mar 20 12:12:01 2020 +0000 + + Fix backport merge error in ESTestCase + + We were ignoring the `stripXContentPosition` boolean when checking for + warnings. + +commit a68071dbbaa5231a61f90b1297f8e3a2eb32a371 +Author: Przemysław Witek +Date: Fri Mar 20 13:08:36 2020 +0100 + + [7.x] Delete empty .ml-state* indices during nightly maintenance task. (#53587) (#53849) + +commit d23112f441a4f0c439534e1d2d2d800531cbf7bd +Author: Alan Woodward +Date: Fri Mar 20 11:40:27 2020 +0000 + + Report parser name and location in XContent deprecation warnings (#53805) + + It's simple to deprecate a field used in an ObjectParser just by adding deprecation + markers to the relevant ParseField objects. The warnings themselves don't currently + have any context - they simply say that a deprecated field has been used, but not + where in the input xcontent it appears. This commit adds the parent object parser + name and XContentLocation to these deprecation messages. + + Note that the context is automatically stripped from warning messages when they + are asserted on by integration tests and REST tests, because randomization of + xcontent type during these tests means that the XContentLocation is not constant + +commit 4e6bbf6e3c5c01b1a74395de2faa486cc21989ad +Author: Jason Tedor +Date: Fri Mar 20 07:34:37 2020 -0400 + + Execute retention lease syncs under system context (#53838) + + The retention lease syncs need to occur under the system context, + because they are internal actions executed on behalf of the user. Today + we are relying on this happening for background syncs by virtue of the + fact that the context the syncs are created under is the system + context. This is due to these occurring on the cluster state applier + thread. However, there are situations where this does not hold such as + when a timed out cluster state publication occurs, and the node where + the shard is allocated is the elected master node. In that case, the + context will be empty due to the fact that we do not reschedule + publication under the system context. Currently, doing so runs us into + some troubles with losing the existing context, possibly dropping + deprecation headers. We could copy that context over when marking the + current context as the system context, but the implications of that + require some more investigation. For now, we explicitly mark the + retention lease syncs as executing under the system context, as this is + situation that we can reason about. + +commit a70ebef366cd9f1070643e1be1de4a7da03e89c0 +Author: Armin Braun +Date: Fri Mar 20 12:23:40 2020 +0100 + + Longer Timeout in S3 Retries Test (#53841) (#53847) + + The lower end of the timeout range of 100ms is prone to time out + on CI before the mock REST server gets to sending a response that + is not supposed to be a timeout. + Using 1-3s here should make this safe at the cost of randomly making + this test take a few seconds. + + Closes #53506 + +commit be3090138e791645b89e40fac52cf516f6d90f4d +Author: lgypro <42293361+lgypro@users.noreply.github.com> +Date: Fri Mar 20 18:44:53 2020 +0800 + + [Docs] Fix typo in _analyze api docs (#53837) + +commit 60153c54335e9cd29044f1a207f506c5837d6433 +Author: Dimitris Athanasiou +Date: Fri Mar 20 12:11:53 2020 +0200 + + [7.x][ML] Data frame analytics analysis stats (#53788) (#53844) + + Adds parsing and indexing of analysis instrumentation stats. + The latest one is also returned from the get-stats API. + + Note that we chose to duplicate objects even where they are currently + similar. There are already ideas on how these will diverge in the future + and while the duplication looks ugly at the moment, it is the option + that offers the highest flexibility. + + Backport of #53788 + +commit f7143b8d85c45cf141115069a1a624e4c8248f1f +Author: Ryan Ernst +Date: Thu Mar 19 15:05:44 2020 -0700 + + Fix Joda compatibility in stream protocol (#53823) + + The JodaCompatibleZonedDateTime is a compatibility object that unions + Joda's DateTime and Java's ZonedDateTime, meant for use in scripts. When + it was added, we serialized the JCZDT as a Joda DateTime so that when + sending to older nodes they could still read the object. However, on + newer nodes, we continued also reading this as a Joda DateTime. This + commit changes the read side to form a JCZDT. + + closes #53586 + +commit 8a0bb6d116cdb08a862ee892de019ad24b22d3fe +Author: Mark Vieira +Date: Thu Mar 19 15:09:02 2020 -0700 + + Enable assertions for IntelliJ junit test runner + +commit f63c32ff31bf0d828b039e29b5c150a11023d2a8 +Author: Ryan Ernst +Date: Thu Mar 19 15:03:47 2020 -0700 + + Make lucene snapshot repository exclusive to lucene packages (#53353) + + When depending on lucene snapshots we point maven at our own s3 backed + repository. However, in this case lucene packages should only be + retrieved from this location, and no other packages should ever be found + in that repo. This commit makes the maven repository exclusive to lucene + packages. + +commit b8ef830c0ac66d2b8b1180ed37072e3fd5fc1894 +Author: Ryan Ernst +Date: Thu Mar 19 14:39:01 2020 -0700 + + Decouple AuditTrailService from AuditTrail (#53450) (#53760) + + The AuditTrailService has historically been an AuditTrail itself, acting + as a composite of the configured audit trails. This commit removes that + interface from the service and instead builds a composite delegating + implementation internally. The service now has a single get() method to + get an AuditTrail implementation which may be called. If auditing is not + allowed by the license, an empty noop version is returned. + +commit c3dee628c71059c7be3cf7e0f370dcb5333b5a06 +Author: Lee Hinman +Date: Thu Mar 19 15:04:00 2020 -0600 + + [7.x] Add IndexTemplateV2 to MetaData (#53753) (#53827) + + * Add IndexTemplateV2 to MetaData (#53753) + + * Add IndexTemplateV2 to MetaData + + This adds the `IndexTemplateV2` and `IndexTemplateV2Metadata` class to be used for the new + implementation of index templates. The new metadata is stored as a `MetaData.Custom` implementation. + + Relates to #53101 + + * Add ITV2Metadata unit tests + + Co-authored-by: Elastic Machine + + * Update min supported version constant + + Co-authored-by: Elastic Machine + +commit 2c77c0df65514512c87ff93c151223d2e0c89c1d +Author: Mayya Sharipova +Date: Thu Mar 19 13:09:50 2020 -0400 + + Fix testIndexhasDuplicateData tests (#49786) + + testIndexHasDuplicateData tests were failing ocassionally, + due to approximate calculation of BKDReader.estimatePointCount, + where if the node is Leaf, the number of points in it + was (maxPointsInLeafNode + 1) / 2. + As DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024, for small indexes + used in tests, the estimation could be really off. + + This rewrites tests, to make the max points in leaf node to + be a small value to control the tests. + + Closes #49703 + +commit 3b2b564c916f3a8d4f4e8a21ebac28788d641c80 +Author: Mark Vieira +Date: Thu Mar 19 11:43:33 2020 -0700 + + Improve IntelliJ IDE integration (#53747) + + This commit makes a number of improvements when importing the + Elasticsearch project into IntelliJ IDEA. Specifically: + + - Contributing documentation has been updated to reflect that the +   'idea' task should no long be used and Gradle project import is +   instead the officially supported way of setting up the project. + - Attempts to run the 'idea' task will result in a failure with a +   message directing folks to our CONTRIBUTING.md document. + - The project JDK is explicit set rather that using whatever JAVA_HOME +   is. + - Gradle build operation delegation is disabled, and test execution is +   configured to 'choose per test'. + - Gradle is configured to inherit the project JDK. + - Some code style conventions are automatically configured. + - File encoding is explicitly set to UTF-8. + - Parallel module compilation is enabled and deprecated feature +   warnings are disabled. + - A remote debug run configuration using listen mode is created. + - JUnit runner is configured with required system properties. + - License headers are configured such that Apache 2 is the default +   notice added to all source files with exception of source in /x-pack +   which will use the Elastic license. + +commit d846ea43f403c2e469cf3dbd491ec18d071c4e0f +Author: Christoph Büscher +Date: Thu Mar 19 19:00:14 2020 +0100 + + Fix ReloadSynonymAnalyzerIT failure (#53663) (#53806) + + There is an assertion in ReloadAnalyzersResponse.merge that compares index names + of merged responses that was falsely using object equality instead of + String.equals(). In the past this didn't seem to matter but with changes in the + test setup we started to see failures. Correcting this and also simplifying test + a bit to be able to run it repeatedly if needed. + + Backport of #53663 + +commit 433952b595e02294453f8da4541551cdabad37d5 +Author: Benjamin Trent +Date: Thu Mar 19 13:56:41 2020 -0400 + + [7.x] [ML] only retry persistence failures when the failure is intermittent and stop retrying when analytics job is stopping (#53725) (#53808) + + * [ML] only retry persistence failures when the failure is intermittent and stop retrying when analytics job is stopping (#53725) + + This fixes two issues: + + + - Results persister would retry actions even if they are not intermittent. An example of an persistent failure is a doc mapping problem. + - Data frame analytics would continue to retry to persist results even after the job is stopped. + + closes https://github.com/elastic/elasticsearch/issues/53687 + +commit cce60215d88763c312ecbf009dee86597f6cfbd8 +Author: Jake Landis +Date: Thu Mar 19 12:29:36 2020 -0500 + + [7.x] Add Watcher to available rest resources (#53620) (#53764) + + Prior to this commit Watcher explicitly copied test between two + projects with a copy task. This commit removes the explicit copy in favor + of adding the Watcher tests to the available restResources that may be + copied between projects. + + This is how inter-project dependencies should be modeled. However, only + Watcher is included here since it is (currently) the only project with + inter-project test dependencies. + +commit db3420d75759d2d66445da9dd25ee9457fde9989 +Author: Jake Landis +Date: Thu Mar 19 12:28:59 2020 -0500 + + [7.x] Optimize which Rest resources are used by the Rest tests… (#53766) + + This should help with Gradle's incremental compile such that projects + only depend upon the resources they use. + + related #52114 + +commit 53f7e314621fb8690364ffaf6a9cd6a83a56f5b5 +Author: István Zoltán Szabó +Date: Thu Mar 19 17:55:26 2020 +0100 + + [DOCS] Fixes typo in start datafeed API docs. (#53811) + +commit 29583288b3333fdafa6ba45cfee3e8eb89c70e20 +Author: István Zoltán Szabó +Date: Thu Mar 19 17:49:19 2020 +0100 + + [DOCS] Adds performance considerations section to transforms overview (#53791) + + Co-Authored-By: Lisa Cawley + +commit 68f7036a4c0b690d8da5eb0c1a47faf98c306dc3 +Author: Lisa Cawley +Date: Thu Mar 19 09:40:29 2020 -0700 + + [DOCS] Adds example links to transform tutorial (#53640) + +commit 40181eb200c5e002edbbee9136cb1d7f0c3d5611 +Author: Lee Hinman +Date: Thu Mar 19 09:35:07 2020 -0600 + + [7.x] Fix feature flag setting for ComponentTemplate APIs (#53… (#53800) + + * Fix feature flag setting for ComponentTemplate APIs (#53758) + + The feature flag was set for *most* of the builds, but there are a couple where it was missing. + + Resolves #53708 + + * Add skip for older versions of ES + +commit 7d3ac4f57d307d696bff0e8122109231ce99ff8b +Author: David Turner +Date: Thu Mar 19 15:20:36 2020 +0000 + + Revert "Apply cluster states in system context (#53785)" + + This reverts commit 4178c57410f406e0b62d522f03d98740956b939e. + +commit 4178c57410f406e0b62d522f03d98740956b939e +Author: David Turner +Date: Thu Mar 19 14:13:52 2020 +0000 + + Apply cluster states in system context (#53785) + + Today cluster states are sometimes (rarely) applied in the default context + rather than system context, which means that any appliers which capture their + contexts cannot do things like remote transport actions when security is + enabled. + + There are at least two ways that we end up applying the cluster state in the + default context: + + 1. locally applying a cluster state that indicates that the master has failed + 2. the elected master times out while waiting for a response from another node + + This commit ensures that cluster states are always applied in the system + context. + + Mitigates #53751 + +commit 00203c35fe15ea5d9b4b42cc503973a3aff08943 +Author: István Zoltán Szabó +Date: Thu Mar 19 15:40:53 2020 +0100 + + [DOCS] Changes seconds to milliseconds since the Epoch in AD docs. (#53797) + +commit dfc1d79ddf3547349da7b14d674779e04ab075af +Author: Ignacio Vera +Date: Thu Mar 19 15:32:09 2020 +0100 + + Add support for distance queries on shape queries (#53468) (#53796) + + With the upgrade to Lucene 8.5, XYShape field has support for distance queries. This change implements this new feature and removes the limitation. + +commit 4f1b2fd2b15f6268fccd2c81f32b7ef77eda8b9d +Author: Ignacio Vera +Date: Thu Mar 19 15:21:58 2020 +0100 + + Add support for distance queries on geo_shape queries (#53466) (#53795) + + With the upgrade to Lucene 8.5, LatLonShape field has support for distance queries. This change implements this new feature and removes the limitation. + +commit b0884baf4665deafc47aba84dd0a476d1dbc5cd1 +Author: Dominic Page <11043991+djptek@users.noreply.github.com> +Date: Thu Mar 19 13:00:36 2020 +0100 + + Geo shape query vs geo point backport (#53774) + + Backport to 7x + + Enable geo_shape query to work on geo_point fields for shapes: circle, polygon, multipolygon, rectangle see: #48928 + Co-Authored-By: @iverase + +commit 8f4a3eb07fbb482e0b05d055f1fae27dbcc910cd +Author: James Rodewig +Date: Thu Mar 19 07:42:26 2020 -0400 + + [DOCS] Add token graph concept docs (#53339) + + Adds conceptual docs for token graphs. + These docs cover: + + * How a token graph is constructed from a token stream + * How synonyms and multi-position tokens impact token graphs + * How token graphs are used during search + * Why some token filters produce invalid token graphs + + Also makes the following supporting changes: + * Adds anchors to the 'Anatomy of an Analyzer' docs for cross-linking + * Adds several SVGs for token graph diagrams + +commit 4b0ae15a9d0b80134cb333e0cf4a2a789f2173f6 +Author: Jim Ferenczi +Date: Thu Mar 19 08:10:34 2020 +0100 + + Disable distributed sort optimization on scroll requests (#53759) + + This commit disables the sort optimization added in #51852 for scroll requests. + Scroll queries keep a state per shard so we cannot modify the request on + the first round (submit). + This bug was introduced in non-released versions which is why this pr + is marked as a non-issue. + +commit 4a36894a48de0ad9b808e5975ecb12a1cbe4221e +Author: Ioannis Kakavas +Date: Thu Mar 19 08:16:23 2020 +0200 + + Mute failing tests (#53781) + + See #53738 + +commit 9b3b08318d32e486ecdb60f4b58f024c00b38dd2 +Author: Mark Vieira +Date: Wed Mar 18 21:07:17 2020 -0700 + + Remove unused import + +commit bc5dae2713b8d57c06ad2395c5dda84139b2c64f +Author: Jason Tedor +Date: Wed Mar 18 22:21:11 2020 -0400 + + Fix compilation in RoutingNode + + This commit fixes compilation in RoutingNode.java after a backport + brought back usage of an API not available in JDK 8. + +commit b649b0d273fd5b088a9a7c59a12c069f18b97ca0 +Author: Leaf-Lin <39002973+Leaf-Lin@users.noreply.github.com> +Date: Thu Mar 19 12:00:01 2020 +1100 + + Update jvm options doc + + There's a typo with extra ` + +commit 90ab949415e6a368e659fcc64249897b0c4c909e +Author: Jason Tedor +Date: Wed Mar 18 20:57:50 2020 -0400 + + Improve performance of shards limits decider (#53577) + + On clusters with a large number of shards, the shards limits allocation + decider can exhibit poor performance leading to timeouts applying + cluster state updates. This occurs because for every shard, we do a loop + to count the number of shards on the node, and the number of shards for + the index of the shard. This is roughly quadratic in the number of + shards. This loop is not necessary, since we already have a O(1) method + to count the number of non-relocating shards on a node, and with this + commit we add some infrastructure to RoutingNode to make counting the + number of shards per index O(1). + +commit 415d73c27d2abf734dc8c9326daf383f846ae046 +Author: Benjamin Trent +Date: Wed Mar 18 19:54:03 2020 -0400 + + [Transform] renamed _cat/transform to _cat/transforms (#53743) (#53771) + + renaming _cat/transform to _cat/transforms for uniformity with the other _cat apis. + +commit 268e512f0b4cb9d16bef0a52cde04cef41d26e2d +Author: Lisa Cawley +Date: Wed Mar 18 15:23:41 2020 -0700 + + [DOCS] Add transform nodes (#53698) + +commit cdbee32f5553727eac3f908aed115a312fb92e8d +Author: Stuart Tettemer +Date: Wed Mar 18 14:44:04 2020 -0600 + + Scripting: Per-context script cache, default off (#52855) (#53756) + + * Adds per context settings: + `script.context.${CONTEXT}.cache_max_size` ~ + `script.cache.max_size` + + `script.context.${CONTEXT}.cache_expire` ~ + `script.cache.expire` + + `script.context.${CONTEXT}.max_compilations_rate` ~ + `script.max_compilations_rate` + + * Context cache is used if: + `script.max_compilations_rate=use-context`. This + value is dynamically updatable, so users can + switch back to the general cache if desired. + + * Settings for context caches take the first value + that applies: + 1) Context specific settings if set, eg + `script.context.ingest.cache_max_size` + 2) Correlated general setting is set to the non-default + value, eg `script.cache.max_size` + 3) Context default + + The reason for 2's inclusion is to allow an easy + transition for users who've customized their general + cache settings. + + Using the general cache settings for the context caches + results in higher effective settings, since they are + multiplied across the number of contexts. So a general + cache max size of 200 will become 200 * # of contexts. + However, this behavior it will avoid users snapping to a + value that is too low for them. + + Backport of: #52855 + Refs: #50152 + +commit 569323928b35e8852ea62f0cbd062d98ea8307c9 +Author: Jason Tedor +Date: Wed Mar 18 16:16:58 2020 -0400 + + Upgrade the bundled JDK to JDK 14 (#53748) + + This commit upgrades the bundled JDK to JDK 14. + +commit 9a328c2b838224367d7549d3a9178df23d935ebc +Author: Christoph Büscher +Date: Wed Mar 18 19:59:30 2020 +0100 + + Add unsupported parameters to HLRC search request (#53745) + + Currently we don't send values for the `pre_filter_shard_size` and + `max_concurrent_shard_requests` SearchRequest parameters over http when using + the High Level Rest Client. This change adds these parameters to the + RequestConverters and tests. + +commit 0e2e06bd7e1452f677497c97cc0fbbba07ad3567 +Author: James Rodewig +Date: Wed Mar 18 14:33:40 2020 -0400 + + [DOCS] Remove incorrect parms from put index template API docs (#53750) + + Removes the `flat_settings` and `timeout` query parameters from the JSON + spec and asciidoc docs for the put index template API. + + These parameters are not supported by the API. + +commit af519cccffc8214eba96f4ab8e931d9751c0c8e4 +Author: Ioannis Kakavas +Date: Wed Mar 18 18:51:06 2020 +0200 + + Revert "Mute TimeSeriesLifecycleActionsIT (#53741)" + + This reverts commit df0ad7569b5edfa4431491c911c34eb01f791446. + +commit 8e17322b3a1ef1d8fa406c0d0dd9d831c1caa832 +Author: Jim Ferenczi +Date: Wed Mar 18 17:20:35 2020 +0100 + + Shortcut query phase using the results of other shards (#51852) (#53659) + + This commit, built on top of #51708, allows to modify shard search requests based on informations collected on other shards. It is intended to speed up sorted queries on time-based indices. For queries that are only interested in the top documents. + + This change will rewrite the shard queries to match none if the bottom sort value computed in prior shards is better than all values in the shard. + For queries that mix top documents and aggregations this change will reset the size of the top documents to 0 instead of rewriting to match none. + This means that we don't need to keep a search context open for this shard since we know in advance that it doesn't contain any competitive hit. + +commit ae19802e29ac8011ffebb1794a1a85803fe11627 +Author: markharwood +Date: Wed Mar 18 15:39:17 2020 +0000 + + Fix highlighter support in PinnedQuery and added test (#53716) (#53729) + + CappedScoreQuery was not delegating queryVisitor calls + + Closes #53699 + +commit df0ad7569b5edfa4431491c911c34eb01f791446 +Author: Ioannis Kakavas +Date: Wed Mar 18 17:38:24 2020 +0200 + + Mute TimeSeriesLifecycleActionsIT (#53741) + + see #53738 + +commit 598d4c1bf946c75516df6fd4b757876eff2ab8a1 +Author: markharwood +Date: Wed Mar 18 15:36:52 2020 +0000 + + Formatting fix + + Bullet points were not rendered correctly + +commit c6e37c766207d0202de138fafe51abf34ba2ded5 +Author: Lisa Cawley +Date: Wed Mar 18 08:29:18 2020 -0700 + + [DOCS] Adds stub for cat transform API (#53737) + +commit d325899c54addc221684eb13eac02d787d6e9143 +Author: Alan Woodward +Date: Wed Mar 18 15:18:49 2020 +0000 + + Use QueryVisitor when extracting PercolatorQuery list for highlighting (#53728) + + The highlighting phase for percolator queries currently uses some custom query + traversal logic to find all instances of PercolatorQuery in the query tree for the + current search context. This commit converts things to instead use a QueryVisitor, + which future-proofs us against new wrapper queries or queries from custom + plugins that the percolator module doesn't know about. + +commit 1615c4b379000b117ea770a46b626afbefec966f +Author: Nhat Nguyen +Date: Wed Mar 18 11:24:19 2020 -0400 + + Fix testKeepTranslogAfterGlobalCheckpoint (#53704) + + Read the global checkpoint after flushed as we might advance it while flushing. + + Closes #53505 + +commit 75c367de1349bf902ce29fc64818c093100957aa +Author: Luca Cavanna +Date: Wed Mar 18 16:15:35 2020 +0100 + + [TEST] Replace agg key in async search yaml test (#53727) + + Some clients have problems running this test as a numeric key is treated like an array index by default. + We can work around this by renaming the aggregation key to not be a numeric. + +commit 2ccb963f1db9f0f68b68d2866ed78bdbfdedbd58 +Author: Benjamin Trent +Date: Wed Mar 18 10:45:28 2020 -0400 + + Create GET _cat/transforms API Issue (#53643) (#53726) + + Adds new` _cat/transform` and `_cat/transform/{transform_id}` endpoints. + +commit 580bc40c0c87b5bdd35d05e250256063c94c1d56 +Author: Alan Woodward +Date: Wed Mar 18 14:15:30 2020 +0000 + + Make it possible to deprecate all variants of a ParseField with no replacement (#53722) + + Sometimes we want to deprecate and remove a ParseField entirely, without replacement; + for example, the various places where we specify a _type field in 7x. Currently we can + tell users only that a particular field name should not be used, and that another name should + be used in its place. This commit adds the ability to say that a field should not be used at + all. + +commit d56dee599a1d8442de93409cfa2b10bf65d40bfa +Author: Marios Trivyzas +Date: Wed Mar 18 14:52:03 2020 +0100 + + Increase step between checks for cancellation (#53712) + + The introduction of the ExitableDirectoryReader showed increase of + latencies for range queries using pointvalues. + + Check for cancellation every 1024 docs instead of every 15 to lower + the impact of the check in query's performance. + + Follows: #52822 + Fixes: #53496 + (cherry picked from commit 6b5fc35e4458e60a7ca5822584ec6a60562f2c01) + +commit 0732475cb9b795f1004dcd9fc5e31b97d0d2fd0a +Author: James Rodewig +Date: Wed Mar 18 09:42:25 2020 -0400 + + [DOCS] Streamline `analyzer` mapping parm def (#51874) + + Simplifies the `analyzer` mapping parameter definition to remove + duplicated analysis content and examples. + +commit e5aa0906f73217cdaff32392bceac381831692f3 +Author: Ioannis Kakavas +Date: Wed Mar 18 14:49:57 2020 +0200 + + Mute testHistoryIsWrittenWithDeletion (#53721) + + see #53718 + +commit e4b7af11ab94b631831bdf139215790d1208ca94 +Author: James Rodewig +Date: Wed Mar 18 08:33:20 2020 -0400 + + [DOCS] Remove `light_bengali` stemmer (#53697) + + Only the `bengali` stemmer is available in Lucene and surfaced through + Elasticsearch. This removes the incorrect `light_bengali` link in our + docs. + +commit 6cc564d677a844f9fb8acab5de37b27d49cb8b25 +Author: Tanguy Leroux +Date: Wed Mar 18 13:23:45 2020 +0100 + + Restore off-heap loading for term dictionary in ReadOnlyEngine (#53713) + + This is a partial restore of #43158, following decision taken in #51247 + + Closes #51247 + +commit 94ac979c66daccda35698bfc37ad75db85b11547 +Author: Dan Hermann +Date: Wed Mar 18 07:07:49 2020 -0500 + + Support array for all string ingest processors (#53694) + +commit 2384c1359d6c4be4b8d5c1be9ea4e227df88b946 +Author: Christoph Büscher +Date: Wed Mar 18 12:44:23 2020 +0100 + + Revert "Fix ReloadSynonymAnalyzerIT failure (#53663)" + + This reverts commit 2c32173fceef472f3a4e9859f6ea3ca6ab3a5a67. + +commit 2c32173fceef472f3a4e9859f6ea3ca6ab3a5a67 +Author: Christoph Büscher +Date: Wed Mar 18 11:55:14 2020 +0100 + + Fix ReloadSynonymAnalyzerIT failure (#53663) + + There is an assertion in ReloadAnalyzersResponse.merge that compares index names + of merged responses that was falsely using object equality instead of + String.equals(). In the past this didn't seem to matter but with changes in the + test setup we started to see failures. Correcting this and also simplifying test + a bit to be able to run it repeatedly if needed. + + Closes #53443 + +commit 474dfbf9c7566a4adf665855587c48da33dcac04 +Author: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> +Date: Wed Mar 18 11:52:50 2020 +0100 + + [Docs] Fix highlighting in match-query example (#52426) + +commit e7ae9ae596ee3d5554bf6375e2e750f7909f0fbd +Author: Tianlun Li +Date: Wed Mar 18 03:03:21 2020 -0700 + + Deprecate delaying state recovery for master nodes (#53646) + + It is useful to be able to delay state recovery until enough data nodes have + joined the cluster, since this gives the shard allocator a decent opportunity + to re-use as much existing data as possible. However we also have the option to + delay state recovery until a certain number of master-eligible nodes have + joined, and this is unnecessary: we require a majority of master-eligible nodes + for state recovery, and there is no advantage in waiting for more. + + This commit deprecates the unnecessary settings in preparation for their + removal. + + Relates #51806 + +commit ec13c093df44a5b5ad92dcc2c0742709722b9d62 +Author: Przemysław Witek +Date: Wed Mar 18 10:28:45 2020 +0100 + + Make ML index aliases hidden (#53160) (#53710) + +commit 85cc7a06c43b7c3d716361c3e98c29c77103bb90 +Author: Yash Joshi +Date: Wed Mar 18 14:42:57 2020 +0530 + + [Docs] Fix typo in range query (#53656) + +commit 873d0ecd091fb48518b9b97bc105555a270534f9 +Author: Ioannis Kakavas +Date: Wed Mar 18 09:43:26 2020 +0200 + + Fix potential bug in concurrent token refresh support (#53668) (#53705) + + Ensure that we do not proceed execution after calling the + listerer's onFailure + +commit 7a12300ce6893c113b08c9706ede56ca6532b738 +Author: Hendrik Muhs +Date: Wed Mar 18 08:37:56 2020 +0100 + + [7.x][Transform] enhance the output of preview to return full… (#53695) + + changes the output format of preview regarding deduced mappings and enhances + it to return all the details about auto-index creation. This allows the user + to customize the index creation. Using HLRC you can create a index request + from the output of the response. + + backport #53572 + +commit a6dca577e5a021653e93fb7d4611a04bb24a3222 +Author: Hendrik Muhs +Date: Tue Mar 17 20:57:54 2020 +0100 + + [Transform] data nanos/date histogram IT (#53654) + + add an integration test for date nanos in combination with date_histogram + +commit 9c0e846db3d7975eca8441914239be87bcb7bd67 +Author: Lee Hinman +Date: Tue Mar 17 13:23:28 2020 -0600 + + [7.x] Add REST API for ComponentTemplate CRUD (#53558) (#53681) + + * Add REST API for ComponentTemplate CRUD + + This adds the Put/Get/DeleteComponentTemplate APIs that allow inserting, retrieving, and removing + ComponentTemplateMetadata into the cluster state metadata. + + These APIs are currently only available behind a feature flag system property - + `es.itv2_feature_flag_registered`. + + Relates to #53101 + + Co-authored-by: Elastic Machine + + Co-authored-by: Elastic Machine + +commit 3c5437894eaacbfa05b1f7cad51b37905be93b40 +Author: Karen Metts <35154725+karenzone@users.noreply.github.com> +Date: Tue Mar 17 14:38:50 2020 -0400 + + Remove link to old settings 7.x (#53639) + +commit 169308656c0fcb9a4b2b1eafe8e10089b95dde99 +Author: Ryan Ernst +Date: Tue Mar 17 11:13:20 2020 -0700 + + Actually add licenses for jackson + + Missed in 1d9f57b + +commit 1d9f57bfc1e0b7d5b746c1a4311713e5486e3108 +Author: Ryan Ernst +Date: Tue Mar 17 10:40:56 2020 -0700 + + Fix databind version reference + + This fixes fallout from a bad backport of #53642 + +commit 5c472fcb47911bc22052a5576ce28431b8f94079 +Author: Ryan Ernst +Date: Tue Mar 17 10:26:35 2020 -0700 + + Upgrade jackson to 2.10.3 and GeoIP to 2.13.1 (#53642) + + Re-applies the change from #53523 along with test fixes. + + closes #53626 + closes #53624 + closes #53622 + closes #53625 + + Co-authored-by: Nik Everett + Co-authored-by: Lee Hinman + Co-authored-by: Jake Landis + +commit 497250ba9cb302b3485faa7ef5c6022837da910e +Author: Mark Vieira +Date: Tue Mar 17 09:57:47 2020 -0700 + + Improve performance of docs snippet length checkstyle rule (#53636) + + The multiline regex rule used to detect docs code snippets greater than + 76 characters in length has considerable cost to. For the high level + rest client project alone, this was taking upwards of 3 minutes. This + commit updates the rule regex pattern to use non-greedy matching when + appropriate which results in a lot fewer backtracks and about a 70% + reduction in execution time on the high level rest client module. + +commit 347b7594b723babc83a56daadbffee1640476dd6 +Author: David Turner +Date: Tue Mar 17 14:04:42 2020 +0000 + + Fix deprecation in history retention docs (#53655) + + This commit adjusts a `deprecation[...]` message in the docs since such + messages must be on a single line. It also moves this message to the start of + the description of the deprecated setting as is the case with other such + messages. + +commit 2b635737e1750a4da8b5e512b62ca44602de4b33 +Author: David Kyle +Date: Tue Mar 17 13:59:52 2020 +0000 + + [ML] Parse single named object in config classes (#53472) (#53542) + +commit 71b703edd1eb2c6ea3f677645000ac9a30d8bb22 +Author: Alan Woodward +Date: Tue Mar 17 12:25:51 2020 +0000 + + Rename AtomicFieldData to LeafFieldData (#53554) + + This conforms with lucene's LeafReader naming convention, and + matches other per-segment structures in elasticsearch. + +commit 79600eb38b81ad0db0ff302043789c97ca841e7b +Author: Andrei Stefan +Date: Tue Mar 17 12:49:08 2020 +0200 + + SQL: add support for index aliases for SYS COLUMNS command (#53525) (#53653) + + (cherry picked from commit f65e4d6ff7b2e00eb6f9c985fbe7cb24de00f045) + +commit a0314ad0150428414973b6a9ff82f5262315cad8 +Author: Hendrik Muhs +Date: Tue Mar 17 11:34:22 2020 +0100 + + [Transform] add transform discovery node role (#53616) + + Enhancement of #52712: Add a discovery node role using the letter t for transform. + + Fixes #53156 + +commit 23af171cf8da7a7d40eaf476e832312bb86aa92b +Author: Ioannis Kakavas +Date: Tue Mar 17 09:45:35 2020 +0200 + + Disallow Password Change when authenticated by Token (#49694) (#53614) + + Password changes are only allowed when the user is currently + authenticated by a realm (that permits the password to be changed) + and not when authenticated by a bearer token or an API key. + +commit 7f21ade92407c5c65d2bd83a42e752838acdde5b +Author: Yang Wang +Date: Tue Mar 17 17:56:37 2020 +1100 + + Explicitly require that derived API keys have no privileges (#53647) (#53648) + + The current implicit behaviour is that when an API keys is used to create another API key, + the child key is created without any privilege. This implicit behaviour is surprising and is + a source of confusion for users. + + This change makes that behaviour explicit. + +commit 74dbdb991ce1707ccb7a4aef8f72212bccf583f2 +Author: Tim Vernum +Date: Tue Mar 17 13:30:07 2020 +1100 + + Avoid NPE in set_security_user without security (#53543) + + If security was disabled (explicitly), then the SecurityContext would + be null, but the set_security_user processor was still registered. + + Attempting to define a pipeline that used that processor would fail + with an (intentional) NPE. This behaviour, introduced in #52032, is a + regression from previous releases where the pipeline was allowed, but + was no usable. + + This change restores the previous behaviour (with a new warning). + + Backport of: #52691 + +commit e7f38674edc3072af15c998d20ddddf345e0d62e +Author: Ryan Ernst +Date: Mon Mar 16 18:54:40 2020 -0700 + + Add internalClusterTest to check task (#53444) + + This commit adds internalClusterTest in xpack core to run as part of + check. This was accidentally removed in a refactoring. Other xpack + modules already do this, but core was left out. This commit also mutes 2 + tests that currently fail. + + closes #53407 + +commit 6a7baddc2b8b8e249211507c980922101a0cedbb +Author: Ryan Ernst +Date: Mon Mar 16 18:50:50 2020 -0700 + + Update to ASM 7.1 in logger usage check (#52742) + + The logger usage check uses its own version of ASM to inspect class + files for logging usages. Master was updated to support java 11 + compilation in #40754. However, 7.x still used ASM 5, which could not + read newer java bytecode versions. This commit bumps ASM in 7.x used in + the logger usage check. + + closes #52408 + +commit 8c96112613b28ad92d68ebbac889ee0410f39aca +Author: Jason Tedor +Date: Mon Mar 16 21:44:58 2020 -0400 + + Fix compilation in simple transport test case base + + This commit fixes compilation after a backport gone bad due to a + difference in the minimum Java version between 7.x and master. + +commit 01d2339883db72bdd2206bb67144c97f3dbdbeb1 +Author: Jason Tedor +Date: Mon Mar 16 21:27:02 2020 -0400 + + Invoke response handler on failure to send (#53631) + + Today it can happen that a transport message fails to send (for example, + because a transport interceptor rejects the request). In this case, the + response handler is never invoked, which can lead to necessary cleanups + not being performed. There are two ways to handle this. One is to expect + every callsite that sends a message to try/catch these exceptions and + handle them appropriately. The other is merely to invoke the response + handler to handle the exception, which is already equipped to handle + transport exceptions. + +commit 881d0bfa8a8b3b9d18e35b57ebcbdffd40144bd3 +Author: Jason Tedor +Date: Mon Mar 16 21:07:55 2020 -0400 + + Add server name to remote info API (#53634) + + This commit adds the configured server_name to the proxy mode info so + that it can be exposed in the remote info API. + +commit c3d241744826dfcf24d788fdbefa3bdfcb1c62a0 +Author: Luca Cavanna +Date: Tue Mar 17 00:08:17 2020 +0100 + + Cumulative backport of async search changes (#53635) + + * Submit async search to work only with POST (#53368) + + Currently the submit async search API can be called using both GET and POST at REST, but given that it submits a call and creates internal state, POST should be the only allowed method. + + * Refine SearchProgressListener internal API (#53373) + + The following cumulative improvements have been made: + - rename `onReduce` and `notifyReduce` to `onFinalReduce` and `notifyFinalReduce` + - add unit test for `SearchShard` + - on* methods in `SearchProgressListener` shouldn't need to be public as they should never be called directly, they only need to be overridden hence they can be made protected. They are actually called directly from a test which required some adapting, like making `AsyncSearchTask.Listener` class package private instead of private + - Instead of overriding `getProgressListener` in `AsyncSearchTask`, as it feels weird to override a getter method, added a specific method that allows to retrieve the Listener directly without needing to cast it. Made the getter and setter for the listener final in the base class. + - rename `SearchProgressListener#searchShards` methods to `buildSearchShards` and make it static given that it accesses no instance members + - make `SearchShard` and `SearchShardTask` classes final + + * Move async search yaml tests to x-pack yaml test folder (#53537) + + The yaml tests for async search currently sit in its qa folder. There is no reason though for them to live in a separate folder as they don't require particular setup. This commit moves them to the main folder together with the other x-pack yaml tests so that they will be run by the client test runners too. + + * [DOCS] Add temporary redirect for async-search (#53454) + + The following API spec files contain a link to a not-yet-created + async search docs page: + + * [async_search.delete.json][0] + * [async_search.get.json][1] + * [async_search.submit.json][2] + + The Elaticsearch-js client uses these spec files to create their docs. + This created a broken link in the Elaticsearch-js docs, which has broken + the docs build. + + This PR adds a temporary redirect for the docs page. This redirect + should be removed when the actual API docs are added. + + [0]: https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json + [1]: https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json + [2]: https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json + + Co-authored-by: James Rodewig + +commit 9845dbb7d6d426dba717fb1310d18d024b303682 +Author: Nik Everett +Date: Mon Mar 16 17:35:43 2020 -0400 + + Fix sorting agg buckets by doc_count (backport of #53617) (#53627) + + I broke sorting aggregations by `doc_count` in #51271 by mixing up true + and false. This flips that comparison and adds a few tests to double + check that we don't so this again. + +commit b4e203a7352aee54e0217c071ded505711b25104 +Author: Ryan Earle +Date: Mon Mar 16 16:41:05 2020 -0400 + + [DOCS] Remove `force` as valid value for `version_type` (#53428) + + The `force` value for the `version_type` parameter was deprecated in 6.8. + + This removes the value from the parameter definition. + +commit 531cd5aaa4cd3b37583c827f3edec9e3e70eb5d8 +Author: Tim Brooks +Date: Mon Mar 9 10:49:41 2020 -0600 + + Add documentation for remote cluster proxy mode (#52779) + + This is related to #49067. + +commit f0beab4041a16a354d6024f270035603dadd7f26 +Author: Nik Everett +Date: Mon Mar 16 16:15:23 2020 -0400 + + Stop using round-tripped PipelineAggregators (backport of #53423) (#53629) + + This begins to clean up how `PipelineAggregator`s and executed. + Previously, we would create the `PipelineAggregator`s on the data nodes + and embed them in the aggregation tree. When it came time to execute the + pipeline aggregation we'd use the `PipelineAggregator`s that were on the + first shard's results. This is inefficient because: + 1. The data node needs to make the `PipelineAggregator` only to + serialize it and then throw it away. + 2. The coordinating node needs to deserialize all of the + `PipelineAggregator`s even though it only needs one of them. + 3. You end up with many `PipelineAggregator` instances when you only + really *need* one per pipeline. + 4. `PipelineAggregator` needs to implement serialization. + + This begins to undo these by building the `PipelineAggregator`s directly + on the coordinating node and using those instead of the + `PipelineAggregator`s in the aggregtion tree. In a follow up change + we'll stop serializing the `PipelineAggregator`s to node versions that + support this behavior. And, one day, we'll be able to remove + `PipelineAggregator` from the aggregation result tree entirely. + + Importantly, this doesn't change how pipeline aggregations are declared + or parsed or requested. They are still part of the `AggregationBuilder` + tree because *that* makes sense. + +commit 278e3fce5091279fae868b29dc5b4cffa4888fa9 +Author: Lisa Cawley +Date: Mon Mar 16 12:14:01 2020 -0700 + + [DOCS] Add anchors for scripted metric aggregations (#53618) + +commit 508e5239b4468c855df54945c1be1b50443bf488 +Author: Abhilash Bolla <2282894+ivssh@users.noreply.github.com> +Date: Tue Mar 17 00:23:53 2020 +0530 + + [DOCS] Remove unneeded anchor from ILM snippet (#53432) + +commit 880cc3ca7eaf5109a63144115f674e241068c0f3 +Author: Gordon Brown +Date: Mon Mar 16 13:07:26 2020 -0600 + + Hide I/SLM history aliases (#53564) + + This commit adjusts the aliases used for the ILM and SLM history indices + to be hidden aliases. + + Also tweaks the configuration of the `IndexTemplateRegistry`s used by + these history system to only upgrade the template from the master node, + as documents are indexed from the master node, so the template version + should only be upgraded from the master node. + +commit f7482f794af1e2a3a6665615ae616c3d85f02376 +Author: Nik Everett +Date: Mon Mar 16 13:47:43 2020 -0400 + + Improve top_metrics docs (#53521) (#53619) + + * Removes experimental. + * Replaces `"v"` (for value) with `"m"` (for metric). + * Move the note about tiebreaking into the list of limitations of the + sort. + * Explain how you ask for `metrics`. + * Clean up some wording. + * Link to the docs from `top_metrics`. + + Closes #51813 + +commit 031932b32f99ed4fb8f7c22f4c0ff0b574287908 +Author: Gordon Brown +Date: Mon Mar 16 11:25:05 2020 -0600 + + Allow _cat indices & aliases to use indices options (#53248) + + This commit adjusts the _cat/indices and _cat/aliases APIs to allow + specifying indices options, so that these APIs can handle hidden + indices/aliases in the same way as other APIs. + + Also adds the hidden option to the expand_wildcards parameter + in the YAML spec for every API that accepts it. + +commit 7571ca437a37859b721cc6eca1d2d16d55dfeff4 +Author: Alexander Reelsen +Date: Mon Mar 16 18:07:14 2020 +0100 + + Disable Watcher script optimization for stored scripts (#53497) + + The watcher TextTemplateEngine uses a fast path mechanism where it + checks for the existence of `{{` to decide if a mustache script + required compilation. This does not work for stored script, as the field + that is checked contains the id of the script, which means, the name of + the script is returned as its value. + + This commit checks for the script type and does not involve this fast + path check if a stored script is used. + + Closes #40212 + +commit 91ca9c5c33a6f017fe1f51438117d9147bc12a25 +Author: Andrei Stefan +Date: Mon Mar 16 18:06:31 2020 +0200 + + QL: constant_keyword support (#53241) (#53602) + + (cherry picked from commit d6cd4ce7849ba215407c8c5fa815c9b373fb8480) + +commit dc2edc97f09e8d914a0fd98a2a9cc75311d9073e +Author: jimczi +Date: Wed Mar 11 18:17:00 2020 +0100 + + Fix sporadic failures in AsyncSearchActionTests (take 2) + + This change removes the need to always get a new version when iterating + on an async search. This is needed since we cannot guarantee that shards will + be queried exactly in order. + + Relates #53360 + +commit e1eebea846d698158f690abb60bbfce53b956e2d +Author: James Rodewig +Date: Mon Mar 16 11:37:06 2020 -0400 + + [DOCS] Reformat `remove_duplicates` token filter (#53608) + + Makes the following changes to the `remove_duplicates` token filter + docs: + + * Rewrites description and adds Lucene link + * Adds detailed analyze example + * Adds custom analyzer example + +commit 2c74f3e22ce8cc8407cc3d38fc41a17a9a24f1fe +Author: markharwood +Date: Mon Mar 16 15:07:13 2020 +0000 + + Backport of new wildcard field type (#53590) + + * New wildcard field optimised for wildcard queries (#49993) + + Indexes values using size 3 ngrams and also stores the full original as a binary doc value. + Wildcard queries operate by using a cheap approximation query on the ngram field followed up by a more expensive verification query using an automaton on the binary doc values. Also supports aggregations and sorting. + +commit a906f8a0e4ae4e67287c3baca82caf82b66147fd +Author: Mayya Sharipova +Date: Mon Mar 16 11:06:25 2020 -0400 + + Highlighters skip ignored keyword values (#53408) (#53604) + + Keyword field values with length more than ignore_above are not + indexed. But highlighters still were retrieving these values + from _source and were trying to highlight them. This sometimes lead to + errors if a field length exceeded max_analyzed_offset. But also this + is an overall wrong behaviour to attempt to highlight something that was + ignored during indexing. + + This PR checks if a keyword value was ignored because of its length, + and if yes, skips highlighting it. + + Backport: #53408 + Closes #43800 + +commit 376b2ae7355671b6d2d78ca141169a101a9eca46 +Author: Przemysław Witek +Date: Mon Mar 16 15:38:56 2020 +0100 + + [7.x] Make classification evaluation metrics work when there is field mapping type mismatch (#53458) (#53601) + +commit e6680be0b10cd5bd447fa381f4eaee2143b5c1fc +Author: Jim Ferenczi +Date: Mon Mar 16 15:31:27 2020 +0100 + + Add new x-pack endpoints to track the progress of a search asynchronously (#49931) (#53591) + + This change introduces a new API in x-pack basic that allows to track the progress of a search. + Users can submit an asynchronous search through a new endpoint called `_async_search` that + works exactly the same as the `_search` endpoint but instead of blocking and returning the final response when available, it returns a response after a provided `wait_for_completion` time. + + ```` + GET my_index_pattern*/_async_search?wait_for_completion=100ms + { + "aggs": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "1h" + } + } + } + ```` + + If after 100ms the final response is not available, a `partial_response` is included in the body: + + ```` + { + "id": "9N3J1m4BgyzUDzqgC15b", + "version": 1, + "is_running": true, + "is_partial": true, + "response": { + "_shards": { + "total": 100, + "successful": 5, + "failed": 0 + }, + "total_hits": { + "value": 1653433, + "relation": "eq" + }, + "aggs": { + ... + } + } + } + ```` + + The partial response contains the total number of requested shards, the number of shards that successfully returned and the number of shards that failed. + It also contains the total hits as well as partial aggregations computed from the successful shards. + To continue to monitor the progress of the search users can call the get `_async_search` API like the following: + + ```` + GET _async_search/9N3J1m4BgyzUDzqgC15b/?wait_for_completion=100ms + ```` + + That returns a new response that can contain the same partial response than the previous call if the search didn't progress, in such case the returned `version` + should be the same. If new partial results are available, the version is incremented and the `partial_response` contains the updated progress. + Finally if the response is fully available while or after waiting for completion, the `partial_response` is replaced by a `response` section that contains the usual _search response: + + ```` + { + "id": "9N3J1m4BgyzUDzqgC15b", + "version": 10, + "is_running": false, + "response": { + "is_partial": false, + ... + } + } + ```` + + Asynchronous search are stored in a restricted index called `.async-search` if they survive (still running) after the initial submit. Each request has a keep alive that defaults to 5 days but this value can be changed/updated any time: + ````` + GET my_index_pattern*/_async_search?wait_for_completion=100ms&keep_alive=10d + ````` + The default can be changed when submitting the search, the example above raises the default value for the search to `10d`. + ````` + GET _async_search/9N3J1m4BgyzUDzqgC15b/?wait_for_completion=100ms&keep_alive=10d + ````` + The time to live for a specific search can be extended when getting the progress/result. In the example above we extend the keep alive to 10 more days. + A background service that runs only on the node that holds the first primary shard of the `async-search` index is responsible for deleting the expired results. It runs every hour but the expiration is also checked by running queries (if they take longer than the keep_alive) and when getting a result. + + Like a normal `_search`, if the http channel that is used to submit a request is closed before getting a response, the search is automatically cancelled. Note that this behavior is only for the submit API, subsequent GET requests will not cancel if they are closed. + + Asynchronous search are not persistent, if the coordinator node crashes or is restarted during the search, the asynchronous search will stop. To know if the search is still running or not the response contains a field called `is_running` that indicates if the task is up or not. It is the responsibility of the user to resume an asynchronous search that didn't reach a final response by re-submitting the query. However final responses and failures are persisted in a system index that allows + to retrieve a response even if the task finishes. + + ```` + DELETE _async_search/9N3J1m4BgyzUDzqgC15b + ```` + + The response is also not stored if the initial submit action returns a final response. This allows to not add any overhead to queries that completes within the initial `wait_for_completion`. + + The `.async-search` index is a restricted index (should be migrated to a system index in +8.0) that is accessible only through the async search APIs. These APIs also ensure that only the user that submitted the initial query can retrieve or delete the running search. Note that admins/superusers would still be able to cancel the search task through the task manager like any other tasks. + + Relates #49091 + + Co-authored-by: Luca Cavanna + +commit 723034001c126fb077420a3018bb336c7dbacf18 +Author: Marios Trivyzas +Date: Mon Mar 16 13:49:48 2020 +0100 + + SQL: Fix NPE for parameterized LIKE/RLIKE (#53573) + + Fix NPE when `null` is passed as a parameter for a parameterized + pattern of LIKE/RLIKE. e.g.: `field LIKE ?` params=[null]` + Check for null pattern in LIKE/RLIKE as for RLIKE (RegexpQuery) we + get an IllegalArgumentExpression from Lucence but for LIKE + (WildcardQuery) we get an NPE. + + Fixes: #53557 + (cherry picked from commit ec3481ed13254ecdec32acf7a0fafd536ec77aff) + +commit 94da4ca3fc8ac98c4f291f4bd0871d8ff0408784 +Author: Dimitris Athanasiou +Date: Mon Mar 16 15:00:54 2020 +0200 + + [7.x][ML] Extend classification to support multiple classes (#53539) (#53597) + + Prepares classification analysis to support more than just + two classes. It introduces a new parameter to the process config + which dictates the `num_classes` to the process. It also + changes the max classes limit to `30` provisionally. + + Backport of #53539 + +commit a38e5ca8e7eb7fd01c2114cd9906f0633f117754 +Author: David Kyle +Date: Mon Mar 16 12:30:56 2020 +0000 + + Mute TimeSeriesLifecycleActionsIT.testHistoryIsWrittenWithFailure (#53595) + + Failure tracked in #50353 + +commit 1272ae411e638d7593c1f2a8779e317472fd6e96 +Author: Marios Trivyzas +Date: Mon Mar 16 11:55:08 2020 +0100 + + SQL: Fix issue with LIKE/RLIKE as painless script (#53495) + + Add missing asScript() implementation for LIKE/RLIKE expressions. + + When LIKE/RLIKE are used for example in GROUP BY or are wrapped with + scalar functions in a WHERE clause, the translation must produce a + painless script which will be executed to implement the correct + behaviour and previously this was completely missing, and as a + consquence wrong results were silently (no error) returned. + + Fixes: #53486 + (cherry picked from commit eaa8ead6742a8e7dcf343bcbaff8de031550fd77) + +commit e2effa9fabcb1ce1b8c99f83cbcfedfd15772878 +Author: bellengao +Date: Fri Mar 13 22:35:02 2020 +0800 + + Fix inaccurate total hit count in _search template api (#53155) + + When 'rest_track_total_hits_as_int' is set to true, the total hits count in the response should be accurate. So we should set trackTotalHits to true if need when parsing the inline script of a search template request. + + Closes #52801 + +commit 3b9545848f4e1749d55112a034ba3250c66971b0 +Author: Martijn van Groningen +Date: Fri Mar 13 15:13:26 2020 +0100 + + Reenable watcher rest tests (#53532) + + Also log a message instead of failing if there are active watches at a beginning of a test. + + Relates to #53177 + +commit 7e82a4f78c7d599ff70ade145d07789989a7b16b +Author: David Turner +Date: Mon Mar 16 08:54:51 2020 +0000 + + Do not log no-op reconnections at DEBUG (#53469) + + Today the NodeConnectionsService emits a DEBUG-level log message each time it + calls TransportService#connectToNode, which happens for every node in the + cluster every ten seconds, and also at every cluster state update. That's a lot + of log messages. Most of these calls are no-ops and can be ignored, but if the + call was not a no-op then it may be worth investigating further. Since the logs + do not distinguish the interesting and uninteresting cases, they are not + useful. + + This commit distinguishes the two cases and pushes the noisy logging for the + common no-op case down to TRACE level, leaving only useful and actionable + information in the DEBUG-level logs. + +commit 2f0aca992bb8c91c17603050807891cad2e41483 +Author: Mark Vieira +Date: Sun Mar 15 18:10:40 2020 -0700 + + Revert "Upgrade to Jackson 2.10.3 and GeoIP2 to 2.13.1 (#53576)" + + This reverts commit b7dbadeea075a409b68122291853587dca529e94. + +commit 66374b61ca7116f7e6419383885f896a8b1fa08f +Author: Jason Tedor +Date: Sat Mar 14 18:11:31 2020 -0400 + + Remove extra code in allocation commands parsing (#53579) + + This commit removes some code that is duplicated in the parsing of + allocation commands in the cluster reroute API. + +commit b7dbadeea075a409b68122291853587dca529e94 +Author: Jason Tedor +Date: Sat Mar 14 13:28:06 2020 -0400 + + Upgrade to Jackson 2.10.3 and GeoIP2 to 2.13.1 (#53576) + + This commit upgrades our Jackson dependency to 2.10.3 and our GeoIP2 + dependency to 2.13.1. + + Relates #53523 + +commit 1262ab2762e1a3aeb0cbbda007c5d536d93c2d3e +Author: Benjamin Trent +Date: Fri Mar 13 16:53:34 2020 -0400 + + [ML] [Inference] fix number inference models returned in x-pack info call (#53540) (#53560) + + the ML portion of the x-pack info API was erroneously counting configuration documents and definition documents. The underlying implementation of our storage separates the two out. + + This PR filters the query so that only trained model config documents are counted. + +commit 4e43ede735fa300c73067089caec4c8ec80a5a94 +Author: Benjamin Trent +Date: Fri Mar 13 15:40:57 2020 -0400 + + [ML] renaming inference processor field field_mappings to new name field_map (#53433) (#53502) + + This renames the `inference` processor configuration field `field_mappings` to `field_map`. + + `field_mappings` is now deprecated. + +commit b6c94fd73e8e5a6f15b8f4fc60b7d739a1f90da8 +Author: Marios Trivyzas +Date: Fri Mar 13 19:26:14 2020 +0100 + + Fix Term Vectors with artificial docs and keyword fields (#53504) (#53550) + + Previously, Term Vectors API was returning empty results for + artificial documents with keyword fields. Checking only for `string()` + on `IndexableField` is not enough, since for `KeywordFieldType` + `binaryValue()` must be used instead. + + Fixes #53494 + + (cherry picked from commit 1fc3fe3d32f41eab2101c0536751b7c47e63cc48) + +commit 690099553c008ebc37576d071bcf22c3f30c5c42 +Author: Tom Veasey +Date: Fri Mar 13 17:35:51 2020 +0000 + + [7.x][ML] Adds the class_assignment_objective parameter to classification (#53552) + + Adds a new parameter for classification that enables choosing whether to assign labels to + maximise accuracy or to maximise the minimum class recall. + + Fixes #52427. + +commit b1d589f2766a3552866c1583ee229394bb83811e +Author: Lisa Cawley +Date: Fri Mar 13 09:32:15 2020 -0700 + + [DOCS] Adds operations_behind to transform stats (#53518) + +commit f8fe1d3344c1471bec8b9cae5c8b7d3b80f91d6f +Author: Marios Trivyzas +Date: Fri Mar 13 16:07:38 2020 +0100 + + Fix YAML test for search.allow_expensive_queries (#53541) (#53548) + + Remove excessive testing and keep only the checks for when the queries + are disallowed. Fix also the check for the initial value of the setting + to be conmbatible with Go client tests. + + (cherry picked from commit 314145294ea926e069c6f8629dfc622a7f31a0fb) + +commit fb29c2dccfca7a043585a43f057c83aa76d7a917 +Author: Dan Hermann +Date: Fri Mar 13 09:41:14 2020 -0500 + + Fix ingest pipeline _simulate api with empty docs never returns a res… (#52937) (#53547) + +commit e7c393a518c01720a463eb68b739834e09ebfc0b +Author: Jim Ferenczi +Date: Fri Mar 13 14:37:20 2020 +0100 + + Fix broken link in client documentation (#51834) + + This change fixes the link to the RankFeatureQueryBuilder javadocs + in the documentation. + + Closes #51800 + +commit 5b718d256551d24c85485eca60a09988fec79c92 +Author: William Brafford +Date: Fri Mar 13 07:49:14 2020 -0400 + + Use snake case for nodes stats/info metric names (#53446) (#53535) + + * Use snake case for nodes stats/info metric names (#53446) + + The REST API uses "thread_pool" as the name of the thread pool metric. + If we use this name internally when we serialize nodes stats and info + requests, we won't need to do any fancy logic to check for and switch + out "threadPool", which was the previous internal name. + +commit 1e31031b738f0dc485abd72cb691f6d552b23e27 +Author: James Rodewig +Date: Fri Mar 13 04:07:40 2020 -0400 + + [DOCS] Clarify `max_shingle_size` parm def (#53480) + + Rewrites the `search_as_you_type` field datatype's `max_shingle_size` + mapping parameter to improve clarity and better communicate trade-offs + regarding index size. + + Relates to [elastic/kibana#55161][0]. + + Closes #51774. + + [0]: https://github.com/elastic/kibana/pull/55161#discussion_r368107177 + +commit a8677499d7b010f805187db6f593b22b942e7faf +Author: Tim Vernum +Date: Fri Mar 13 16:30:20 2020 +1100 + + [Backport] Add support for secondary authentication (#53530) + + This change makes it possible to send secondary authentication + credentials to select endpoints that need to perform a single action + in the context of two users. + + Typically this need arises when a server process needs to call an + endpoint that users should not (or might not) have direct access to, + but some part of that action must be performed using the logged-in + user's identity. + + Backport of: #52093 + +commit bac1740d448acba571fe64f1ec1dad987a702b66 +Author: Tim Vernum +Date: Fri Mar 13 14:27:29 2020 +1100 + + Support authentication without anonymous user (#53528) + + This change adds a new parameter to the authenticate methods in the + AuthenticationService to optionally exclude support for the anonymous + user (if an anonymous user exists). + + Backport of: #52094 + +commit 9dfcc07401a6a1c555cbd8cf65b7dc9b7c7d15f9 +Author: Jim Ferenczi +Date: Fri Mar 13 01:27:26 2020 +0100 + + Fix pre-sorting of shards in the can_match phase (#53397) + + This commit fixes a bug on sorted queries with a primary sort field + that uses different types in the requested indices. In this scenario + the returned min/max values to sort the shards are not comparable so + we should avoid the sorting rather than throwing an obscure exception. + +commit f696360517b09ce400f97fc0916bd42e028a08f2 +Author: Jason Tedor +Date: Thu Mar 12 20:23:49 2020 -0400 + + Fix SHAs for :x-pack:snapshot-tool + + This commit fixes the SHA for jackson-databind in :x-pack:snapshot-tool. + +commit 32dd852210274c5986a8b43e5a3d5f0477b203eb +Author: Jason Tedor +Date: Thu Mar 12 20:14:30 2020 -0400 + + Update jackson-databind to 2.8.11.6 (#53522) + + This commit upgrades the jackson-databind depdendency to + 2.8.11.6. Additionally, we revert a previous change that put + ingest-geoip on the version of jackson-databind from the version + properties file. This is because upgrading ingest-geoip to a later + version of jackson-databind also requires an upgrade to the geoip2 + dependency which is currently blocked. Therefore, if we can get to a + point where we otherwise upgrade our Jackson dependencies, we do not + want ingest-geoip to automatically come along with it. + +commit fe2f6b359e647a059708f6f5dba1eb90b1f96210 +Author: Nhat Nguyen +Date: Thu Mar 12 11:51:16 2020 -0400 + + Fix concurrent requests race over scroll context limit (#53449) + + Concurrent search scroll requests can lead to more scroll contexts than the limit. + +commit 2789fe4179021335393176c1011befe74607c8df +Author: Lee Hinman +Date: Thu Mar 12 15:33:32 2020 -0600 + + [7.x] Add ComponentTemplate to MetaData (#53290) (#53489) + + * Add ComponentTemplate to MetaData (#53290) + + * Add ComponentTemplate to MetaData + + This adds a `ComponentTemplate` datastructure that will be used as part of #53101 (Index Templates + v2) to the `MetaData` class. Currently there are no APIs for interacting with this class, so it will + always be an empty map (other than in tests). This infrastructure will be built upon to add APIs in + a subsequent commit. + + A `ComponentTemplate` is made up of a `Template`, a version, and a MetaData.Custom class. The + `Template` contains similar information to an `IndexTemplateMetaData` object— settings, mappings, + and alias configuration. + + * Update minimal supported version constant + + Co-authored-by: Elastic Machine + +commit 9dcd64c1108386d830e3b5bc8f7a25497a9b46fd +Author: Nik Everett +Date: Thu Mar 12 17:17:09 2020 -0400 + + Preserve metric types in top_metrics (backport of #53288) (#53440) + + This changes the `top_metrics` aggregation to return metrics in their + original type. Since it only supports numerics, that means that dates, + longs, and doubles will come back as stored, with their appropriate + formatter applied. + +commit 97621e7f65911c0f818098b2df573ecbad6e88dc +Author: Jim Ferenczi +Date: Thu Mar 12 21:17:11 2020 +0100 + + Removes old Lucene's experimental flag from analyzer documentations (#53217) + + This change removes the Lucene's experimental flag from the documentations of the following + tokenizer/filters: + * Simple Pattern Split Tokenizer + * Simple Pattern tokenizer + * Flatten Graph Token Filter + * Word Delimiter Graph Token Filter + + The flag is still present in Lucene codebase but we're fully supporting these tokenizers/filters + in ES for a long time now so the docs flag is misleading. + + Co-authored-by: James Rodewig + +commit 0353b804bf4717746a48e441d6040f1347f6b75e +Author: Jay Modi +Date: Thu Mar 12 13:08:13 2020 -0600 + + Mute testKeepTranslogAfterGlobalCheckpoint (#53510) + + This change mutes a test that fails reproducibly in + InternalEngineTests. + + Relates #53505 + +commit 5b08ea84c9ee3008e69df0260f9cb1fb50bbc911 +Author: Jason Tedor +Date: Thu Mar 12 14:32:41 2020 -0400 + + Add deprecation check for listener thread pool (#53438) + + This commit adds a deprecation check for the listener thread pool + settings as these will be removed in 8.0.0. + +commit 3972cbeba989bc4cc81ce6e549da2ba948ac8a38 +Author: Mark Vieira +Date: Thu Mar 12 10:10:40 2020 -0700 + + Remove Java 13 from runtime testing matrix + + With Java 14 soon to be GA 13 will be EOL so there's no need to continue + testing on it. + + Signed-off-by: Mark Vieira + +commit 969164cc47211deb3c2d0c000e89666ebe05fcca +Author: István Zoltán Szabó +Date: Thu Mar 12 18:00:36 2020 +0100 + + [DOCS] Adds a warning about reindexing docs with the same ID to the PUT DFA docs. (#53490) + +commit 67fffe676e6dc82b5c2f12097c825f1fc1df1ba2 +Author: Lee Hinman +Date: Thu Mar 12 10:59:31 2020 -0600 + + [7.x] Add read/writeOptionalVLong to StreamInput/Output (#5314… (#53491) + + The spirit of StreamInput/StreamOutput is that common I/O patterns should + be handled by these classes so that the persistence methods in + application classes can be kept short, which facilitates easy visual + comparison between read and write methods, and reduces risks of having + serialization issues due to mismatched implementations. + + To this end, this change adds readOptionalVLong and writeOptionalVLong + methods to these classes as we have started to build up cases where + that conditional/null logic has been implemented directly in the read & + write methods. + + Co-authored-by: Tim Vernum + +commit af36665b08a513e4fc8046b22e5e7bf17902dde5 +Author: Jay Modi +Date: Thu Mar 12 10:18:39 2020 -0600 + + Deprecate the logstash enabled setting (#53487) + + The setting, `xpack.logstash.enabled`, exists to enable or disable the + logstash extensions found within x-pack. In practice, this setting had + no effect on the functionality of the extension. Given this, the + setting is now deprecated in preparation for removal. + + Backport of #53367 + +commit 34adfd9611b9a9e3ea7660c15f737ca1b555f5b5 +Author: Dan Hermann +Date: Thu Mar 12 10:14:51 2020 -0500 + + Validate SSL settings at parse time (#49196) (#53473) + +commit 31d45b3c956d1fbca9d4c7b68255bb0db7649b49 +Author: Aleksandr Maus +Date: Thu Mar 12 10:58:07 2020 -0400 + + EQL: Improve query folder test suite (#53187) (#53476) + + Related to https://github.com/elastic/elasticsearch/issues/52775 + +commit e3a1291adf4a0b8798b31ab0e7bb54dd7f0092fe +Author: Aleksandr Maus +Date: Thu Mar 12 09:44:46 2020 -0400 + + EQL: Add more rest client tests (#53422) (#53474) + +commit 9face1be3874ac05c6a23b496c5e3bc7be27793c +Author: David Kyle +Date: Thu Mar 12 13:21:36 2020 +0000 + + [7.x] Add ObjectParser.declareNamedObject (singular) method (#53017) (#53395) + + Add the convenience method AbstractObjectParser.declareNamedObject (singular) to + complement the existing declareNamedObjects (plural). + +commit 48124807d55a2ae057484179bcff9e758ae1f90a +Author: Yannick Welsch +Date: Thu Mar 12 14:14:36 2020 +0100 + + Fix SourceOnlySnapshotIT (#53462) + + The tests in this class had been failing for a while, but went unnoticed as not tested by CI (see #53442). + + The reason the tests fail is that the can-match phase is smarter now, and filters out access to a non-existing field. + + Closes #53442 + +commit af987fb2d40a3ea24efee330c7a867ddfa858bae +Author: James Rodewig +Date: Thu Mar 12 05:57:23 2020 -0400 + + [DOCS] Reduce content reuse in enrich docs (#53460) + + Restructures the 'Update an enrich policy' section to: + + * Migrate the content to the section. It was previously stored in the + Put Enrich Policy API docs. + * Remove the warning tag admonition from the section content. + * Replace a reused section earlier in the "Set up an enrich processor" + page with a link. + + No substantive changes were made to the content. + +commit 2438b899eb0b1bdf03746bb764e67ea86ca40f08 +Author: Przemyslaw Gomulka +Date: Thu Mar 12 08:57:03 2020 +0100 + + Support joda style date patterns in 7.x (#52555) + + If an index was created in version 6 and contain a date field with a joda-style pattern it should still be allowed to search and insert document into it. + Those created in 6 but date pattern starts with 8, should be considered as java style. + +commit 4301c357835aa13e64ad7224e0664ee4a37eea4c +Author: Ryan Ernst +Date: Wed Mar 11 15:45:47 2020 -0700 + + Use exclusiveContent gradle feature for jdk and distro downloads (#53358) + + The jdk and distribution download plugins create fake ivy repositories, + and use group based repository filtering to ensure no other artifacts + try to resolve against the fake repos. Currently this works by adding a + blanket exclude to all repositories for the given group name. This + commit changes to using the new exclusiveContent feature in + Gradle to do the exclusion. + +commit d8e70d468867119927d5e44f1e0fc5b6d8e5d24f +Author: Jason Tedor +Date: Wed Mar 11 16:44:47 2020 -0400 + + Enable deprecation checks for removed settings (#53317) + + Today we do not have any infrastructure for adding a deprecation check + for settings that are removed. This commit enables this by adding such + infrastructure. Note that this infrastructure is unused in this commit, + which is deliberate. However, the primary target for this commit is 7.x + where this infrastructue will be used, in a follow-up. + +commit 89668c5ea015daec5dd830814427a06f940c5c3d +Author: Benjamin Trent +Date: Wed Mar 11 13:49:39 2020 -0400 + + [ML][Inference] adds new default_field_map field to trained models (#53294) (#53419) + + Adds a new `default_field_map` field to trained model config objects. + + This allows the model creator to supply field map if it knows that there should be some map for inference to work directly against the training data. + + The use case internally is having analytics jobs supply a field mapping for multi-field fields. This allows us to use the model "out of the box" on data where we trained on `foo.keyword` but the `_source` only references `foo`. + +commit 9ada50834798073a7374cd0741c7ec43b82382b3 +Author: Nik Everett +Date: Wed Mar 11 13:00:07 2020 -0400 + + Fix date_nanos in composite aggs (backport of #53315) (#53347) + + It looks like `date_nanos` fields weren't likely to work properly in + composite aggs because composites iterate field values using points and + we weren't converting the points into milliseconds. Because the doc + values were coming back in milliseconds we ended up geting very confused + and just never collecting sub-aggregations. + + This fixes that by adding a method to `DateFieldMapper.Resolution` to + `parsePointAsMillis` which is similarly in name and function to + `NumberFieldMapper.NumberType`'s `parsePoint` except that it normalizes + to milliseconds which is what aggs need at the moment. + + Closes #53168 + +commit c408a34a212512b09aab8ab7c363dbd10cf1cbda +Author: Lisa Cawley +Date: Wed Mar 11 09:13:25 2020 -0700 + + [DOCS] Fixes link to custom realm examples (#53205) + +commit 9a21a8abf2b93eb928bfa730a05e00717f491eca +Author: Jay Modi +Date: Wed Mar 11 09:58:37 2020 -0600 + + Opt-in logstash plugin to formatting (#53413) + + This change opts-in the logstash plugin for enforced formatting. + + Backport of #53370 + +commit 1fd56698fa2e0002f58610f6c06cc935fbf8c803 +Author: Nhat Nguyen +Date: Wed Mar 11 08:34:19 2020 -0400 + + Adjust wire version for search context id + + Relates #53143 + +commit 6665ebe7ab2534dc1f2aff13749ef383eb9f2af0 +Author: Nhat Nguyen +Date: Mon Mar 9 11:59:30 2020 -0400 + + Harden search context id (#53143) + + Using a Long alone is not strong enough for the id of search contexts + because we reset the id generator whenever a data node is restarted. + This can lead to two issues: + + 1. Fetch phase can fetch documents from another index + 2. A scroll search can return documents from another index + + This commit avoids these issues by adding a UUID to SearchContexId. + +commit 8c4c19d310e31d587243747645f77f8f3bddaed5 +Author: Przemysław Witek +Date: Wed Mar 11 15:36:38 2020 +0100 + + Perform evaluation in multiple steps when necessary (#53295) (#53409) + +commit 562a9eff33e65a801b43c3a0f8861cac659f5fef +Author: Ignacio Vera +Date: Wed Mar 11 14:04:22 2020 +0100 + + remove sneaked placeholder from histogram docs (#53391) (#53405) + +commit 933a9c6fca3d7c476be312f2449aab82b3ac8dd5 +Author: James Rodewig +Date: Wed Mar 11 09:03:57 2020 -0400 + + [DOCS] Reformat `word_delimiter` token filter (#53387) + + Makes the following changes to the `word_delimiter` token filter docs: + + * Adds a warning admonition recommending the `word_delimiter_graph` + filter instead. This warning includes a link to the deprecated Lucene + `WordDelimiterFilter`. + * Updates the description + * Adds detailed analyze snippet + * Adds custom analyzer and custom filter snippets + * Reorganizes and updates parameter documentation + +commit 063957b7d8bffb97a9aac4a2f15d527c7fcec3cd +Author: Przemysław Witek +Date: Wed Mar 11 12:26:11 2020 +0100 + + Simplify "refresh" calls. (#53385) (#53393) + +commit cc7751eb161d9894a0a0765b59c657fb2c69f200 +Author: Dimitris Athanasiou +Date: Wed Mar 11 13:01:34 2020 +0200 + + [7.x][ML] Add ILM policy to ml stats indices (#53349) (#53392) + + Adds a size based ILM policy to automatically + rollover ml stats indices. + + Backport of #53349 + +commit 0fd0516d0d1f8b370e2d68a2f9bc586c2497d167 +Author: Dimitris Athanasiou +Date: Wed Mar 11 12:45:27 2020 +0200 + + [7.x][ML] Rename data frame analytics maximum_number_trees to max_trees (#53300) (#53390) + + Deprecates `maximum_number_trees` parameter of classification and + regression and replaces it with `max_trees`. + + Backport of #53300 + +commit 532a720e1bedb769a18ba7c437132f64f2dfc4dc +Author: David Roberts +Date: Wed Mar 11 10:20:00 2020 +0000 + + [ML] Skeleton estimate_model_memory endpoint for anomaly detection (#53386) + + This is a partial implementation of an endpoint for anomaly + detector model memory estimation. + + It is not complete, lacking docs, HLRC and sensible numbers + for many anomaly detector configurations. These will be + added in a followup PR in time for 7.7 feature freeze. + + A skeleton endpoint is useful now because it allows work on + the UI side of the change to commence. The skeleton endpoint + handles the same cases that the old UI code used to handle, + and produces very similar estimates for these cases. + + Backport of #53333 + +commit ac721938c2276144ed4d2a301c898e8a9cabd705 +Author: David Turner +Date: Wed Mar 11 09:03:54 2020 +0000 + + Allow joining node to trigger term bump (#53338) + + In rare circumstances it is possible for an isolated node to have a greater + term than the currently-elected leader. Today such a node will attempt to join + the cluster but will not offer a vote to the leader and will reject its cluster + state publications due to their stale term. This situation persists since there + is no mechanism for the joining node to inform the leader that its term is + stale and a new election is required. + + This commit adds the current term of the joining node to the join request. Once + the join has been validated, the leader will perform another election to + increase its term far enough to allow the isolated node to join properly. + + Fixes #53271 + +commit a9dd7773d27fe1a754477223c04532c7a6b38682 +Author: James Rodewig +Date: Wed Mar 11 04:45:26 2020 -0400 + + [DOCS] Use keyword tokenizer in word delimiter graph examples (#53384) + + In a tip admonition, we recommend using the `keyword` tokenizer with the + `word_delimiter_graph` token filter. However, we only use the + `whitespace` tokenizer in the example snippets. This updates those + snippets to use the `keyword` tokenizer instead. + + Also corrects several spacing issues for arrays in these docs. + +commit 7189c57b6cbc07014b16f18942172b7d38d089ad +Author: Armin Braun +Date: Wed Mar 11 06:30:36 2020 +0100 + + Record Force Merges in Live Commit Data (#52694) (#53372) + + * Record Force Merges in live commit data + + Prerequisite of #52182. Record force merges in the live commit data + so two shard states with the same sequence number that differ only in whether + or not they have been force merged can be distinguished when creating snapshots. + +commit 2ab502afc4a9c74fe0fe691d338904fa45bb628e +Author: Jake Landis +Date: Tue Mar 10 20:57:29 2020 -0500 + + [7.x] Remove dead 'beats' code (#53312) (#53376) + +commit 24f114766fb1dae17bb94146042d14152386882c +Author: Nhat Nguyen +Date: Tue Mar 10 19:17:39 2020 -0400 + + Fix doc_stats and segment_stats of ReadOnlyEngine (#53345) + + We can't always have the same segment stats and doc stats between + InternalEngine and ReadOnlyEngine if there are some fully deleted + segments. ReadOnlyEngine always filters out them. InternalEngine, + however, will keep them if peer recovery retention leases exist or the + number of the retaining operations is non-zero. + + This change reverts the fix in #51331 and uses the wrapped reader to + calculate the segment stats and doc stats. For the test, we need to + disable the extra retaining soft-deletes operations. + + Closes #51303 + +commit cad02d4a31e34041dcccf9649e503b81fd5338ec +Author: Nhat Nguyen +Date: Tue Mar 3 10:23:24 2020 -0500 + + Increase timeout testFollowIndexMaxOperationSizeInBytes (#53014) + + Replicating 1000 documents one by one (as we cap the request size at + 1 byte) can take more than 10 seconds on a slow CI. + + Closes #52812 + +commit 3494c73c8d15d8d18ec27ce0305bf3cf3fbd70a0 +Author: William Brafford +Date: Tue Mar 10 16:01:31 2020 -0400 + + Mute failing tests (#53362) (#53363) + +commit 847ac9c7d788774e52070fda3e82f2c7d9b912ca +Author: Przemko Robakowski +Date: Tue Mar 10 20:44:30 2020 +0100 + + Fix null config in SnapshotLifecyclePolicy.toRequest (#53328) (#53355) + + This avoids NPE when executing SLM policy when no config was provided. + + Related to #44465 + + Closes #53171 + + Co-authored-by: Elastic Machine + + Co-authored-by: Elastic Machine + +commit 20bbe5bae4e84b5a42fc71e3bde88906e6ba716e +Author: Gordon Brown +Date: Tue Mar 10 10:56:12 2020 -0600 + + Fix Rollover handing of hidden aliases (#53146) + + Prior to this commit, rollover did not propagate the `is_hidden` alias + property when rollover over an index. This commit ensures that an alias + that's rollover over will remain hidden. + +commit 5ce6de2c1a8b6734f46b4f3a1662e752aff3dff2 +Author: Nik Everett +Date: Tue Mar 10 11:39:10 2020 -0400 + + Simplify SiblingPipelineAggregator (#53144) (#53341) + + This removes the `instanceof`s from `SiblingPipelineAggregator` by + adding a `rewriteBuckets` method to `InternalAggregation` that can be + called to, well, rewrite the buckets. The default implementation of + `rewriteBuckets` throws the same exception that was thrown when you + attempted to run a `SiblingPipelineAggregator` on an aggregation without + buckets. It is overridden by `InternalSingleBucketAggregation` and + `InternalMultiBucketAggregation` to correctly rewrite their buckets. + +commit 89c0e1f566e242018df6f0dcec810c660fa46245 +Author: Nik Everett +Date: Tue Mar 10 11:32:46 2020 -0400 + + Fix composite agg sort bug (backport of #53296) (#53337) + + When an composite aggregation is run against an index with a sort that + *starts* with the "source" fields from the composite but has additional + fields it'd blow up in while trying to decide if it could use the sort. + This changes it to decide that it *can* use the sort. + + Closes #52480 + +commit 166b5a92f685ad93035c0015b51098728dc66a55 +Author: James Rodewig +Date: Tue Mar 10 10:15:42 2020 -0400 + + [DOCS] Correct anchor in word delimiter graph token filter docs + +commit ae6c25b749ec706678b8da06a88a63c67ec4c0f5 +Author: Jim Ferenczi +Date: Tue Mar 10 13:24:21 2020 +0100 + + Speed up partial reduce of terms aggregations (#53216) + + This change optimizes the merge of terms aggregations by removing + the priority queue that was used to collect all the buckets during + a non-final reduction. We don't need to keep the result sorted since + the merge of buckets in a subsequent reduce can modify the order. + I wrote a small micro-benchmark to test the change and the speed ups + are significative for small merge buffer sizes: + + ```` + ########## Master: + Benchmark (bufferSize) (cardinality) (numShards) (topNSize) Mode Cnt Score Error Units + TermsReduceBenchmark.reduceTopHits 5 10000 1000 1000 avgt 10 2459,690 ± 198,682 ms/op + TermsReduceBenchmark.reduceTopHits 16 10000 1000 1000 avgt 10 1030,620 ± 91,544 ms/op + TermsReduceBenchmark.reduceTopHits 32 10000 1000 1000 avgt 10 558,608 ± 44,915 ms/op + TermsReduceBenchmark.reduceTopHits 128 10000 1000 1000 avgt 10 287,333 ± 8,342 ms/op + TermsReduceBenchmark.reduceTopHits 512 10000 1000 1000 avgt 10 257,325 ± 54,515 ms/op + + ########## Patch: + Benchmark (bufferSize) (cardinality) (numShards) (topNSize) Mode Cnt Score Error Units + TermsReduceBenchmark.reduceTopHits 5 10000 1000 1000 avgt 10 805,611 ± 14,630 ms/op + TermsReduceBenchmark.reduceTopHits 16 10000 1000 1000 avgt 10 378,851 ± 17,929 ms/op + TermsReduceBenchmark.reduceTopHits 32 10000 1000 1000 avgt 10 261,094 ± 10,176 ms/op + TermsReduceBenchmark.reduceTopHits 128 10000 1000 1000 avgt 10 241,051 ± 19,558 ms/op + TermsReduceBenchmark.reduceTopHits 512 10000 1000 1000 avgt 10 231,643 ± 6,170 ms/op + ```` + + The code for the benchmark can be found [here](). It seems to be up to 3x faster for terms aggregations + that return 10,000 unique terms (1000 terms per shard). For a cardinality of 100,000 terms, this patch is up to 5x faster: + + ```` + ########## Patch: + Benchmark (bufferSize) (cardinality) (numShards) (topNSize) Mode Cnt Score Error Units + TermsReduceBenchmark.reduceTopHits 5 100000 1000 1000 avgt 10 12791,083 ± 397,128 ms/op + TermsReduceBenchmark.reduceTopHits 16 100000 1000 1000 avgt 10 3974,939 ± 324,617 ms/op + TermsReduceBenchmark.reduceTopHits 32 100000 1000 1000 avgt 10 2186,285 ± 267,124 ms/op + TermsReduceBenchmark.reduceTopHits 128 100000 1000 1000 avgt 10 914,657 ± 160,784 ms/op + TermsReduceBenchmark.reduceTopHits 512 100000 1000 1000 avgt 10 604,198 ± 145,457 ms/op + + ########## Master: + Benchmark (bufferSize) (cardinality) (numShards) (topNSize) Mode Cnt Score Error Units + TermsReduceBenchmark.reduceTopHits 5 100000 1000 1000 avgt 10 60696,107 ± 929,944 ms/op + TermsReduceBenchmark.reduceTopHits 16 100000 1000 1000 avgt 10 16292,894 ± 783,398 ms/op + TermsReduceBenchmark.reduceTopHits 32 100000 1000 1000 avgt 10 7705,444 ± 77,588 ms/op + TermsReduceBenchmark.reduceTopHits 128 100000 1000 1000 avgt 10 2156,685 ± 88,795 ms/op + TermsReduceBenchmark.reduceTopHits 512 100000 1000 1000 avgt 10 760,273 ± 53,738 ms/op + ```` + + The merge of buckets can also be optimized. Currently we use an hash map to merge buckets coming from different shards so this can be costly if the number of unique terms is high. Instead, we could always sort the shard terms result by key and perform a merge sort to reduce the results. This would save memory and make the merge more linear in terms + of complexity in the coordinating node at the expense of an additional sort in the shards. I plan to test this possible optimization in a follow up. + + Relates #51857 + +commit d54d7f2be04b0c4069eedd00a1c576afe8fa8080 +Author: Przemysław Witek +Date: Tue Mar 10 14:24:18 2020 +0100 + + [7.x] Implement ILM policy for .ml-state* indices (#52356) (#53327) + +commit e23c3f915f11bbe9d1ddbdb2fa0fc222e6c33033 +Author: Nik Everett +Date: Tue Mar 10 09:22:19 2020 -0400 + + Save a little space on empty BitArrays (#53243) (#53316) + + It doesn't make a whole lot of sense for `BitArray#clear` to grow the + underlying storage array just to clear the bit. We *already* treat + indices outside of the storage array as unset. This turns such + operations into a noop. + +commit 856d9bfbc1807c0e7187d0e5ebad8371972c07f3 +Author: Benjamin Trent +Date: Tue Mar 10 08:30:47 2020 -0400 + + [ML] fixing data frame analysis test when two jobs are started in succession quickly (#53192) (#53332) + + A previous change (#53029) is causing analysis jobs to wait for certain indices to be made available. While this it is good for jobs to wait, they could fail early on _start. + + This change will cause the persistent task to continually retry node assignment when the failure is due to shards not being available. + + If the shards are not available by the time `timeout` is reached by the predicate, it is treated as a _start failure and the task is canceled. + + For tasks seeking a new assignment after a node failure, that behavior is unchanged. + + + closes #53188 + +commit 5912895838ee5582da520563093e33f7067d0b80 +Author: Hendrik Muhs +Date: Tue Mar 10 13:21:09 2020 +0100 + + [Transform] wait for transform templates in Rest integration t… (#53330) + + add transform templates to the list of templates to be installed before + executing tests + +commit d503bf9a452bc8370f0612abbf92b317fba0d91d +Author: James Rodewig +Date: Tue Mar 10 06:52:30 2020 -0400 + + [DOCS] Make token graph diagrams consistent (#53331) + + Updates the SVG for a token graph to make the layout consistent with + other graphs. This means moving the text directly above the edge lines. + Previously, the text was above the edge line. + +commit 696aa4ddafb2b5e1ef3e7660778cd343dc8d1ace +Author: Hendrik Muhs +Date: Tue Mar 10 11:12:58 2020 +0100 + + [7.x][Transform] add support for script in group_by (#53167) (#53324) + + add the possibility to base the group_by on the output of a script. + + closes #43152 + backport #53167 + +commit 5c861cfe6e2b9e303af525ba487bda95ed65414d +Author: Alan Woodward +Date: Tue Mar 10 09:30:37 2020 +0000 + + Upgrade to final lucene 8.5.0 snapshot (#53293) + + Lucene 8.5.0 release candidates are imminent. This commit upgrades master to use + the latest snapshot to check that there are no last-minute bugs or regressions. + +commit 5e3df18d56d29cf635c662376ab992871c8d2ee6 +Author: James Rodewig +Date: Tue Mar 10 05:14:55 2020 -0400 + + [DOCS] Adds Beats tip to EQL search docs (#53292) + + Adds a tip admonition to the basic example in the EQL search docs. + + This tip lets users know they can set up a Beat to automatically + index data in ES, rather than manually indexing using the bulk or index + APIs. + +commit 641eb69520d4c55422ae062d00fed5d4f6cbb382 +Author: James Rodewig +Date: Tue Mar 10 05:03:17 2020 -0400 + + [DOCS] Document `nodes` cluster stats (#52813) + + Documents the `nodes` response parameters returned by the + `_cluster/stats` API. + + Also adds collapsible attributes for the `indices` and `nodes` + sections. + +commit b68d7b1c331e559ac5558aab2df0ea4822b7842e +Author: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com> +Date: Tue Mar 10 08:50:38 2020 +0000 + + giving kibana user privileges to create custom link index (#53221) (#53278) + +commit a4d481f2bbc32c0ded663afdd64622a0a699e55d +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Mar 10 08:02:02 2020 +0100 + + ILM Freeze step retry when not acknowledged (#53287) + + A freeze operation can partially fail in multiple places, including the + close verification step. This left the index in an unfrozen but + partially closed state. Now throw an exception to retry the freeze step + instead. + +commit 1cb0a4399d4c6cab13902b7fc89021166e949b84 +Author: Gordon Brown +Date: Mon Mar 9 16:16:29 2020 -0600 + + Fix Get Alias API handling of hidden indices with visible aliases (#53147) + + This commit changes the Get Aliases API to include hidden indices by + default - this is slightly different from other APIs, but is necessary + to make this API work intuitively. + +commit 2bb4b96a7fac949a4ffea7d9454cff4da058d25b +Author: William Brafford +Date: Mon Mar 9 18:13:29 2020 -0400 + + Serialize NodesStatsRequest as set of strings (#53235) (#53313) + + * Add unit tests before refactoring + + * Convert boolean fields to set of strings + + In order to make nodes stats plugins pluggable, we need to make the + NodesStatsRequest class capable of carrying a flexible list of metrics + rather than a fixed list of boolean flags. This commit changes the + internal storage of the class without changing its serialization. + + * Change serialization of NodesStatsRequest + + * Set up BWC before merging + + * Singularize enum name + +commit f075d70cf8f523c87f6d8c9dea794b270239ea55 +Author: Przemko Robakowski +Date: Mon Mar 9 22:24:15 2020 +0100 + + [7.x] Avoid race condition in ILMHistorySotre (#53039) (#53094) + + * Avoid race condition in ILMHistorySotre (#53039) + + * Avoid race condition in ILMHistorySotre + + This change modifies ILMHistoryStore to always apply correct settings and mappings, + even if template is deleted and not yet recreated. This ensures that ILM history index + is correctly managed by ILM and also fixes flaky history tests that were prone to + triggenring this race. + + This commit also refactors and simplifies ILM history tests. + + Closes #50353 and #52853 + + * Review comment + + Co-authored-by: Elastic Machine + + * fixed tests + + * backport #53306 + + Co-authored-by: Elastic Machine + +commit 1860c5714713f9efd358f7a433b9acff5f68a307 +Author: Jason Tedor +Date: Mon Mar 9 16:50:32 2020 -0400 + + Deprecate the listener thread pool (#53266) + + The listener thread pool is being removed from use in the server + codebase. This commit deprecates configuring the listener thread pool. + +commit 62c8ac99932e51e900234b71a9df0b013fb4c37e +Author: Bogdan Pintea +Date: Mon Mar 9 21:16:57 2020 +0100 + + SQL: transfer version compatibility decision to the server (#53082) (#53302) + + This commit adds a new request object field, "version", containing the version of the requesting client. This parameter is now accepted - and for certain clients required - by the server and the request is validated against it. Currently server's and client's versions still need to be equal in order for the request to be accepted. Relaxing this check is going to be part of future work. + + On the clients' side, the only check remaining is to ensure that the peer server is supporting version backwards compatibility (i.e. is on, or newer than a certain release). + + (cherry picked from commit a8f413a20fb023bec83af0de1211a2936a7f558c) + +commit d064846416584ee5d84c3e79c9315d7f7a7f2156 +Author: Aleksandr Maus +Date: Mon Mar 9 14:11:54 2020 -0400 + + EQL: Test infrastructure improvements (#53253) (#53297) + + Update CommonEqlRestTestCase code to simplify making changes as requested. + Update EqlActionIT to simplify the test code as requested. + Replace Jackson parser with XContent in EqlActionIT. + Whitelist more EQL tests specs that are now supported. + +commit c33afea9fbdf6165b811d1df72afe964a172e85e +Author: Julie Tibshirani +Date: Mon Mar 9 10:57:23 2020 -0700 + + Small corrections to stored_fields docs. (#53247) + + * Fix a reference to the 'field' option. + * Remove claim about detecting script fields. + * Specify that object fields will just be ignored. + +commit f5f922c6f681b78091877909c829b2a1749e01f1 +Author: Ross Wolf <31489089+rw-access@users.noreply.github.com> +Date: Mon Mar 9 10:40:20 2020 -0600 + + EQL: Add IsNull/IsNotNull checks (#52791) + + * EQL: Add IsNull/IsNotNull checks + * EQL: Simplify IsNull/IsNotNull optimization + * EQL: Split string tests over multiple lines + +commit f78bd00f57ecde94d26cef640483c6ce4ad00a7b +Author: Dominik Riemer +Date: Mon Mar 9 16:05:49 2020 +0100 + + [DOCS] Update StreamPipes link in Integrations docs (#53087) + + Updates the link to [StreamPipes][0] after transition to Apache Software Foundation. + + [0]: https://streampipes.apache.org + +commit b20f86e4500ec63b24e98ff0404e03d818e34a23 +Author: David Turner +Date: Mon Mar 9 14:43:58 2020 +0000 + + Clarify JavaDoc for DiscoveryNodes#resolveNodes (#53277) + + Closes #52887 + +commit 64ea94ac8e0d25476297932fb296fa8654958d45 +Author: Greg Back <1045796+gtback@users.noreply.github.com> +Date: Mon Mar 9 09:33:34 2020 -0500 + + [DOCS] Update ES Python client doc links (#53092) + + Replaces `rtfd.org` links with `readthedocs.org`, which is now the official URL. + +commit 8ad0080a59768e31349178957c92ef21acd0502f +Author: Jason Tedor +Date: Mon Mar 9 08:54:53 2020 -0400 + + Fork CCR checkpoint listeners on CCR thread pool (#53265) + + This commit moves the global checkpoint listeners used in CCR to the CCR + thread pool. This removes the last use of the listener thread pool in + the codebase. + +commit 00657901eced439c3c17b334873fdfd3af5491f4 +Author: Romain Gonord +Date: Mon Mar 9 13:40:00 2020 +0100 + + [DOCS] Fix "Wait for Snapshot" link in ILM actions reference (#53269) + + Corrects an anchor for the `Wait for Snapshot` action, which previously linked to the `Delete` action. + +commit 35c8226419b02b5adc302c32f817caa54929bd2a +Author: Anton Dollmaier +Date: Mon Mar 9 13:15:07 2020 +0100 + + [DOCS] Fix parameter formatting for GeoHash grid agg docs (#53032) + + Adds missing colon (`:`) to the parameter definition list. + +commit 52ff341814f6b840c780902e706071fea80dadf7 +Author: David Turner +Date: Mon Mar 9 10:11:17 2020 +0000 + + Deprecate passing settings in restore requests (#53268) + + Today we accept a `settings` field in snapshot restore requests, but this field + is not used. This commit deprecates it. + +commit 2fd954a3b7abfe80cf702d58e5f5a43da44c5521 +Author: Christoph Büscher +Date: Mon Mar 9 12:16:09 2020 +0100 + + Fix potential NPE in FuzzyTermsEnum (#53231) + + Under certain circumstances SpanMultiTermQueryWrapper uses + SpanBooleanQueryRewriteWithMaxClause as its rewrite method, which in turn tries + to get a TermsEnum from the wrapped MultiTermQuery currently using a `null` + AttributeSource. While queries TermsQuery or subclasses of AutomatonQuery ignore + this argument, FuzzyQuery uses it to create a FuzzyTermsEnum which triggers an + NPE when the AttributeSource is not provided. This PR fixes this by supplying an + empty AttributeSource instead of a `null` value. + + Closes #52894 + +commit 28cb4a167df9f85919e948c501ad6057e989ce21 +Author: James Rodewig +Date: Mon Mar 9 06:45:44 2020 -0400 + + [DOCS] Reformat `word_delimiter_graph` token filter (#53170) (#53272) + + Makes the following changes to the `word_delimiter_graph` token filter + docs: + + * Updates the Lucene experimental admonition. + * Updates description + * Adds analyze snippet + * Adds custom analyzer and custom filter snippets + * Reorganizes and updates parameter list + * Expands and updates section re: differences between `word_delimiter` + and `word_delimiter_graph` + +commit aafc0409a9ed87cf54f2c0de3a9932f8ebfd8c4d +Author: István Zoltán Szabó +Date: Mon Mar 9 09:48:23 2020 +0100 + + [DOCS] Makes the description clearer on how to use aggregations in an anomaly detection job (#53103) + + Co-authored-by: lcawl + +commit 7775ddbc9c7550dc9f68dae75c870d6f1e70ccd6 +Author: Martijn van Groningen +Date: Fri Mar 6 14:53:19 2020 +0100 + + Verify watch_count before a test starts and not after a test. + + This check was added as part of: 0f2d26bdcae370a1225e58e06585907c5c4082eb + + Checking this before the test starts makes more sense, because + the watches index has then also be removed. + + Relates to #53177 + +commit 5e96d3e59aeff620257ceea460b8d46741b7f46d +Author: Jason Tedor +Date: Sun Mar 8 13:48:09 2020 -0400 + + Use given executor for global checkpoint listener (#53260) + + Today when notifying a global checkpoint listener, we use the listener + thread pool. This commit turns this inside out so that the global + checkpoint listener must provide an executor on which to notify the + listener. + +commit 79b67eb3baa4bba8575b5763a6171571ddfbcb1e +Author: Jason Tedor +Date: Sun Mar 8 12:34:59 2020 -0400 + + Drop action future that forks on listener executor (#53261) + + This commit drops the dispatching listenable action future that forks to + the listener thread pool. This was previously used in the transport + client but is no longer used. + +commit a0b235888fba95fcfb6789620d43ef95c92364ea +Author: Jason Tedor +Date: Sun Mar 8 08:58:59 2020 -0400 + + Avoid self-suppression on grouped action listener (#53262) + + It can be that a failure is repeated to a grouped action listener. For + example, if the same exception such as a connect transport exception, is + the cause of repeated failures. Previously we were unconditionally + self-suppressing the exception into the first exception, but + self-supressing is not allowed. Thus, we would throw an exception and + the grouped action listener would never complete. This commit addresses + this by guarding against self-suppression. + +commit c5738ae312a14906748046cc3a62f9c08868ff30 +Author: Jason Tedor +Date: Sat Mar 7 13:02:02 2020 -0500 + + Notify refresh listeners on the calling thread (#53259) + + Today we notify refresh listeners by forking to the listener thread pool + and then serially notifying listeners on a thread there. Refreshes are + expensive though, so the expectation is that we are executing refreshes + on threads that can afford an expensive operation (e.g., not a network + thread) and as such, executing listeners that we expect to be cheap aon + the calling thread is okay. This commit removes the forking of notifying + refresh listeners to run directly on the calling thread that executed a + refresh. + +commit 341417613e5fe3df847a73ae039146c88b689272 +Author: Lisa Cawley +Date: Fri Mar 6 16:28:54 2020 -0800 + + [7.x][DOCS] Adds common definitions for security settings (#51017) (#53242) + + Co-Authored-By: Tim Vernum + +commit ff9b8bda63f397982af2365f1e9c22ef87e89f94 +Author: Gordon Brown +Date: Fri Mar 6 16:02:38 2020 -0700 + + Implement hidden aliases (#52547) + + This commit introduces hidden aliases. These are similar to hidden + indices, in that they are not visible by default, unless explicitly + specified by name or by indicating that hidden indices/aliases are + desired. + + The new alias property, `is_hidden` is implemented similarly to + `is_write_index`, except that it must be consistent across all indices + with a given alias - that is, all indices with a given alias must + specify the alias as either hidden, or all specify it as non-hidden, + either explicitly or by omitting the `is_hidden` property. + +commit 7c9641ef9d2f1115f1e2f54dc09697625d4dd85a +Author: Nik Everett +Date: Fri Mar 6 15:27:51 2020 -0500 + + Simplify BucketedSort (#53199) (#53240) + + Our lovely `BitArray` compactly stores "flags", lazilly growing its + underlying storage. It is super useful when you need to store one bit of + data for a zillion buckets or a documents or something. Usefully, it + defaults to `false`. But there is a wrinkle! If you ask it whether or + not a bit is set but it hasn't grown its underlying storage array + "around" that index then it'll throw an `ArrayIndexOutOfBoundsException`. + The per-document use cases tend to show up in order and don't tend to + mind this too much. But the use case in aggregations, the per-bucket use + case, does. Because buckets are collected out of order all the time. + + This changes `BitArray` so it'll return `false` if the index is too big + for the underlying storage. After all, that index *can't* have been set + or else we would have grown the underlying array. Logically, I believe + this makes sense. And it makes my life easy. At the cost of three lines. + + *but* this adds an extra test to every call to `get`. I think this is + likely ok because it is "very close" to an array index lookup that + already runs the same test. So I *think* it'll end up merged with the + array bounds check. + +commit d6813cb348e1034fc1af61bc2d22e62d207a004a +Author: Ross Wolf <31489089+rw-access@users.noreply.github.com> +Date: Fri Mar 6 13:11:59 2020 -0700 + + EQL: Convert wildcards to LIKE in analyzer (#51901) + + * EQL: Convert wildcard comparisons to Like + * EQL: Simplify wildcard handling, update tests + * EQL: Lint fixes for Optimizer.java + +commit f96ad5c32d580336f7e2701c70d1c1c62e511eb5 +Author: Mayya Sharipova +Date: Fri Mar 6 12:46:31 2020 -0500 + + Mute testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows + +commit a81460dbf5a78ef6f7307928b8ddd030bd2cff89 +Author: Jay Modi +Date: Fri Mar 6 09:47:03 2020 -0700 + + Make watch history indices hidden (#52974) + + This commit updates the template used for watch history indices with + the hidden index setting so that new indices will be created as hidden. + + Relates #50251 + Backport of #52962 + +commit a88d0c7ca3959e3057923fea545875bbd0dd60ed +Author: Adam Canady +Date: Fri Mar 6 11:16:41 2020 -0500 + + [Docs] Correct examples for * and + in regexp-syntax.asciidoc (#53210) + +commit 09a3f4588077d6c7c6efcbb8568eacf92a0f9874 +Author: Mark Vieira +Date: Fri Mar 6 07:37:35 2020 -0800 + + Mute ClassificationIT.testTwoJobsWithSameRandomizeSeedUseSameTrainingSet + + Signed-off-by: Mark Vieira + +commit 01f00df5cd45d2b26418c0024816448dda723446 +Author: James Baiera +Date: Thu Mar 5 15:28:41 2020 -0500 + + Mute RegressionIT.testTwoJobsWithSameRandomizeSeedUseSameTrainingSet + +commit 85d7112e78c7d865ab1fc5ef92ca80ad114a2516 +Author: Benjamin Trent +Date: Fri Mar 6 10:27:21 2020 -0500 + + [ML] Fixing datafeed bwc tests (#52959) + + Datafeed bwc tests have been muted for some time in the 7.x. This is because of date_histogram interval deprecation warnings. + + This commit fixes the tests as must as possible while still handling deprecation warnings. + +commit bf3dcd42296954e3d72b730d02088565e4aaf9b8 +Author: István Zoltán Szabó +Date: Fri Mar 6 16:03:09 2020 +0100 + + [DOCS] Adds deleting flag to the GET job stats API docs (#53223) + +commit 9e561c2921e0340ad54e930cf56b3c4498fb7934 +Author: Christoph Büscher +Date: Fri Mar 6 14:55:48 2020 +0100 + + Fix AbstractBulkByScrollRequest slices parameter via Rest (#53068) + + Currently the AbstractBulkByScrollRequest accepts slice values of 0 via its + `setSlices` method, denoting the "auto" slicing behaviour that is usable by + settting the "slices=auto" parameter on rest requests. When using the High Level + Rest Client, however, we send the 0 value as an integer, which is then rejected + as invalid by `AbstractBulkByScrollRequest#parseSlices`. Instead of making + parsing of the rest request more lenient, this PR opts for changing the + RequestConverter logic in the client to translate 0 values to "auto" on the rest + requests. + + Closes #53044 + +commit d145b5536f08d96539642f82d432a58ae6dca849 +Author: William Brafford +Date: Fri Mar 6 09:07:49 2020 -0500 + + Serialize NodesInfoRequest as a set of strings (#53140) (#53202) + + For Node Info to be pluggable, NodesInfoRequest must be able to carry + arbitrary strings. This commit reworks the internals of that class to + use a set rather than hard-coded boolean fields. + + NodesInfoRequest defaults to specifying all values. We test for + this behavior as we refactor and use random testing for the + various combinations of metrics. + + Add backwards compatibility for transport requests. + +commit 9bb9f6336452f3f060f35f51049254215fae0ee0 +Author: James Rodewig +Date: Fri Mar 6 07:27:14 2020 -0500 + + [DOCS] Note that `trim` filter doesn't change offsets (#53220) + + The [word delimiter graph token filter docs][0] note that the `trim` + filter changes the length of tokens without changing their offsets. + + This explicitly mentions that in the `trim` filter docs. + + [0]: https://www.elastic.co/guide/en/elasticsearch/reference/master/analysis-word-delimiter-graph-tokenfilter.html + +commit 9abf53752710ba63fd565e855bee6bce0283b36e +Author: Dimitris Athanasiou +Date: Fri Mar 6 13:47:27 2020 +0200 + + [7.x][ML] Improve DF analytics audits and logging (#53179) (#53218) + + Adds audits for when the job starts reindexing, loading data, + analyzing, writing results. Also adds some info logging. + + Backport of #53179 + +commit 7ddbda4c20a8dceb882f5470303fa2999ee05508 +Author: Marios Trivyzas +Date: Fri Mar 6 11:04:01 2020 +0100 + + Check for query cancellation during rewrite (#53166) (#53203) + + With ExitableDirectoryReader in place, check for query cancellation + during QueryPhase#preProcess where the query rewriting takes place. + + Follows: #52822 + + (cherry picked from commit 0d38626d8e6e9e2620a7a446b617a2ac42852461) + +commit c204137451902d06fd1636adb6ea7dc55a954077 +Author: Alan Woodward +Date: Fri Mar 6 09:10:56 2020 +0000 + + Deprecate BoolQueryBuilder's mustNot field (#53125) + + The bool query builder in elasticsearch accepts both must_not and mustNot + fields. Given that leniency is abhorrent and must be eschewed, we should deprecate + the latter as it doesn't fit with the style of parameters elsewhere in the DSL. + +commit 2e924e4a83f1d157786c4148464cfb0ee8a3e178 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Mar 6 08:56:27 2020 +0100 + + Fix ClusterDisruptionIT.testAckedIndexing (#53169) + + Use assertBusy when doing reroute after bridged disruption, + since it can return non-acked if a node is marked faulty + by follower check after disruption ended. + + Closes #53064 + +commit 6a5d9195aa6b50ccdcd5ce4e1231b05eae91868d +Author: Jake Landis +Date: Thu Mar 5 17:41:17 2020 -0600 + + [7.x] Ensure only plugin REST tests are run for plugins (#5318… (#53196) + + This commit fixes ensures that for external builds + (e.g. plugin development) that the REST tests that are + copied are properly filtered to only include the API + by default. + + The code prior to this change resulted in including both + the API and tests since the copy.include resulted as an + empty list by default since the stream is empty unless + explicitly configured. + + related #52114 + fixes #53183 + +commit 5476a49833c5c230017e51ab5d207f33dd1343f2 +Author: Nhat Nguyen +Date: Thu Mar 5 17:33:00 2020 -0500 + + Revert "upgrade to lucene-snapshot-fa75139efea (#53150) (#53151)" + + This reverts commit 058113aa424e56f7449810915dc12807149eca1f. + +commit d456e8ffca09752ff414afe7cee719a77b29742a +Author: Nhat Nguyen +Date: Thu Mar 5 17:32:18 2020 -0500 + + Revert "Mute InternalEngineTests.testVersionOnPrimaryWithConcurrentRefresh" + + This reverts commit 66788afa67463c138ffcc02926d7b24de31a97d0. + +commit e9e209ae5827cac6e41d57b027a8fdb4798317e8 +Author: Nhat Nguyen +Date: Thu Mar 5 17:32:11 2020 -0500 + + Revert "Mute InternalEngineTests.testRandomOperations" + + This reverts commit d1cc2e68d5ac6efbd67c768977e920dfa6265870. + +commit dc78cc613145d224bf8d8a37d25288d46d080b3d +Author: Nhat Nguyen +Date: Thu Mar 5 17:31:56 2020 -0500 + + Revert "Mute InternalEngineTests.testForceMergeWithSoftDeletesRetentionAndRecoverySource" + + This reverts commit da8aac9e66a2dd022889ea95228c8a4b0c927cb6. + +commit f11ae5fd148aadcc23ef2d7468e9c3b9dd6c13d3 +Author: Nhat Nguyen +Date: Thu Mar 5 17:31:38 2020 -0500 + + Revert "Mute GatewayMetaStatePersistedStateTests.testDataOnlyNodePersistence" + + This reverts commit 4452addf10f87794296a13117760c91341ee4acf. + +commit f32e4583d1baf83cac3178e4b261838e08528edd +Author: Nik Everett +Date: Thu Mar 5 17:11:54 2020 -0500 + + Add `allowed_warnings` to yaml tests (backport of #53139) (#53173) + + When we test backwards compatibility we often end up in a situation + where we *sometimes* get a warning, and sometimes don't. Like, we won't + get the warning if we're testing against an older version, but we will + in a newer one. Or we won't get the warning if the request randomly + lands on a node with an old version of the code. But we wouldn't if it + randomed into a node with newer code. + + This adds `allowed_warnings` to our yaml test runner for those cases: + warnings declared this way are "allowed" but not "required". + + Blocks #52959 + + Co-authored-by: Benjamin Trent + +commit 4452addf10f87794296a13117760c91341ee4acf +Author: James Baiera +Date: Thu Mar 5 16:43:37 2020 -0500 + + Mute GatewayMetaStatePersistedStateTests.testDataOnlyNodePersistence + +commit da8aac9e66a2dd022889ea95228c8a4b0c927cb6 +Author: James Baiera +Date: Thu Mar 5 15:55:50 2020 -0500 + + Mute InternalEngineTests.testForceMergeWithSoftDeletesRetentionAndRecoverySource + +commit d1cc2e68d5ac6efbd67c768977e920dfa6265870 +Author: James Baiera +Date: Thu Mar 5 15:08:46 2020 -0500 + + Mute InternalEngineTests.testRandomOperations + +commit 66788afa67463c138ffcc02926d7b24de31a97d0 +Author: James Baiera +Date: Thu Mar 5 15:06:10 2020 -0500 + + Mute InternalEngineTests.testVersionOnPrimaryWithConcurrentRefresh + +commit 4bc6d2dbec3aaf8fd5edab77a60275062cbaca0e +Author: James Rodewig +Date: Thu Mar 5 14:48:27 2020 -0500 + + [DOCS] Correct link for Lucene StopFilter + +commit 7e2a9f58eed5bcc6afe702ba113a339700128383 +Author: Mayya Sharipova +Date: Thu Mar 5 14:23:39 2020 -0500 + + script_score query errors on negative scores (#53133) + + 7.5 and 7.6 had a regression that allowed for + script_score queries to have negative scores. + We have corrected this regression in #52478. + This is an addition to #52478 that adds + a test and release notes. + +commit 8851fb2a081d9a889d4502366f2831556cfdbf05 +Author: Mark Vieira +Date: Thu Mar 5 09:26:37 2020 -0800 + + Upgrade to Gradle 6.2.2 (#53136) + +commit c2627aa22f287fe1da05a5524e9bd451a40202d2 +Author: David Turner +Date: Thu Mar 5 15:27:56 2020 +0000 + + Clarify futher the order for a rolling upgrade (#52964) + + Expands the "master-ineligible then master-eligible" sentence into a list and + specifies that within these subsets the order doesn't matter. + +commit 58ce56f6c88a7131be8424d9924f7164868c56a0 +Author: István Zoltán Szabó +Date: Thu Mar 5 16:25:43 2020 +0100 + + [DOCS] Makes the naming convention of the DFA response objects coherent (#53172) + +commit af0b1c2860bc9cdaa56edc8ccc5b830ca165e0b8 +Author: Benjamin Trent +Date: Thu Mar 5 09:59:18 2020 -0500 + + [ML] Fix minor race condition in dataframe analytics _stop (#53029) (#53164) + + Tests have been periodically failing due to a race condition on checking a recently `STOPPED` task's state. The `.ml-state` index is not created until the task has already been transitioned to `STARTED`. This allows the `_start` API call to return. But, if a user (or test) immediately attempts to `_stop` that job, the job could stop and the task removed BEFORE the `.ml-state|stats` indices are created/updated. + + This change moves towards the task cleaning up itself in its main execution thread. `stop` flips the flag of the task to `isStopping` and now we check `isStopping` at every necessary method. Allowing the task to gracefully stop. + + closes #53007 + +commit 181ee3ae0b69dc2fc84d557c417a62ed09fdac92 +Author: Benjamin Trent +Date: Thu Mar 5 09:50:52 2020 -0500 + + [ML] specifying missing_field_value value and using it instead of empty_string (#53108) (#53165) + + For analytics, we need a consistent way of indicating when a value is missing. Inheriting from anomaly detection, analysis sent `""` when a field is missing. This works fine with numbers, but the underlying analytics process actually treats `""` as a category in categorical values. + + Consequently, you end up with this situation in the resulting model + ``` + { + "frequency_encoding" : { + "field" : "RainToday", + "feature_name" : "RainToday_frequency", + "frequency_map" : { + "" : 0.009844409027270245, + "No" : 0.6472019970785184, + "Yes" : 0.6472019970785184 + } + } + } + ``` + For inference this is a problem, because inference will treat missing values as `null`. And thus not include them on the infer call against the model. + + This PR takes advantage of our new `missing_field_value` option and supplies `\0` as the value. + +commit 48707ec55ac4a539f36b0e45108d46791717d954 +Author: István Zoltán Szabó +Date: Thu Mar 5 15:30:30 2020 +0100 + + [DOCS] Expands GET DFA stat API docs with response objects. (#53107) + +commit 2dc872f052d7c07a3adeca6e0497fc5facc58991 +Author: Aleksandr Maus +Date: Thu Mar 5 09:20:38 2020 -0500 + + EQL: Add HLRC for EQL stats (#53043) (#53148) + +commit 360ac1997f92e9440f9b6301dad2527d1749e6e2 +Author: Adrien Grand +Date: Thu Mar 5 14:28:44 2020 +0100 + + Fix test failures with the new `constant_keyword` field. (#53153) + + This test failed because YAML tests randomly install an index template + that updates the default number of shards to 2. + + Closes #53131 + +commit 487d44276064226e18f99517423aaed880a7d45e +Author: Marios Trivyzas +Date: Thu Mar 5 14:17:31 2020 +0100 + + Implement Exitable DirectoryReader (#52822) (#53162) + + Implement an Exitable DirectoryReader that wraps the original + DirectoryReader so that when a search task is cancelled the + DirectoryReaders also stop their work fast. This is usuful for + expensive operations like wilcard/prefix queries where the + DirectoryReaders can spend lots of time and consume resources, + as previously their work wouldn't stop even though the original + search task was cancelled (e.g. because of timeout or dropped client + connection). + + (cherry picked from commit 67acaf61f33bc5f54e26541514d07e375c202e03) + +commit 28df7ae5edb7029a9340699e4f86e1a839466819 +Author: Nik Everett +Date: Thu Mar 5 08:12:01 2020 -0500 + + Support multiple metrics in `top_metrics` agg (backport of #52965) (#53163) + + This adds support for returning multiple metrics to the `top_metrics` + agg. It looks like: + ``` + POST /test/_search?filter_path=aggregations + { + "aggs": { + "tm": { + "top_metrics": { + "metrics": [ + {"field": "v"}, + {"field": "m"} + ], + "sort": {"s": "desc"} + } + } + } + } + ``` + +commit 01504df876ac2d1ad06ee65470df86ba0625e1a3 +Author: David Roberts +Date: Thu Mar 5 09:43:56 2020 +0000 + + [TEST] Force close failed job before skipping test (#53128) + + The assumption added in #52631 skips a problematic test + if it fails to create the required conditions for the + scenario it is supposed to be testing. (This happens + very rarely.) + + However, before skipping the test it needs to remove the + failed job it has created because the standard test + cleanup code treats failed jobs as fatal errors. + + Closes #52608 + +commit 204c366a4e5f8b64cf0e07c8e84621ef4f823bac +Author: Armin Braun +Date: Thu Mar 5 11:18:18 2020 +0100 + + Upgrade GCS SDK to 1.104.0 (#52839) (#53152) + + Upgrading the GCS SDK to the most recent version. + Adjusting (i.e. improving) the REST mock accordingly. + This should significantly boost performance by pulling in + https://github.com/googleapis/java-core/issues/86 in some cases. + +commit 3cd4b97618402bd75d505a020a7e5cc4b54a808e +Author: Alan Woodward +Date: Thu Mar 5 09:48:11 2020 +0000 + + Remove UnknownNamedObjectException (#53105) + + This was originally thrown from NamedXContentRegistry#parseNamedObject() but + that method now throws a NamedObjectNotFoundException, so this is unused. + +commit e46bb54c7b99ca2d1802eaa3a7a9fd0e6e9d1fca +Author: James Rodewig +Date: Thu Mar 5 05:02:47 2020 -0500 + + [DOCS] Document `any` keyword in EQL syntax (#52821) (#53157) + + Adds documentation for the `any` keyword to the EQL syntax docs. + + Includes: + + * Definition of an event category and its relationship to the event + category field. + * Example matching all event categories using `any` keyword + * Example using `any` with `where true` + +commit 058113aa424e56f7449810915dc12807149eca1f +Author: Ignacio Vera +Date: Thu Mar 5 10:04:05 2020 +0100 + + upgrade to lucene-snapshot-fa75139efea (#53150) (#53151) + +commit 52d4807f8d95b35c07a97bf779a690562e222fc0 +Author: Tanguy Leroux +Date: Thu Mar 5 09:18:05 2020 +0100 + + Mute GoogleCloudStorageBlobStoreRepositoryTests on jdk8 (#53119) + + Tests in GoogleCloudStorageBlobStoreRepositoryTests are known + to be flaky on JDK 8 (#51446, #52430 ) and we suspect a JDK + bug (https://bugs.openjdk.java.net/browse/JDK-8180754) that triggers + some assertion on the server side logic that emulates the Google + Cloud Storage service. + + Sadly we were not able to reproduce the failures, even when using + the same OS (Debian 9, Ubuntu 16.04) and JDK (Oracle Corporation + 1.8.0_241 [Java HotSpot(TM) 64-Bit Server VM 25.241-b07]) of + almost all the test failures on CI. While we spent some time fixing + code (#51933, #52431) to circumvent the JDK bug they are still flaky + on JDK-8. This commit mute these tests for JDK-8 only. + + Close ##52906 + +commit 859c6441b3a33d4573d890bf0c46282de3f60199 +Author: Lisa Cawley +Date: Wed Mar 4 14:42:03 2020 -0800 + + [DOCS] Adds PKI delegation.enabled example (#53030) + +commit 302980e0c47366bb02705465e047b2c2d0e20d5c +Author: Nik Everett +Date: Wed Mar 4 13:06:41 2020 -0500 + + Remove some ceremony in agg parsing (#53078) (#53117) + + With #50871 aggrgations should now be parsed directly by an + `ObjectParser` or `ConstructingObjectParser` without the need for the + ceremonial `parse` method. This removes 9 of those `parse` methods and + parses the aggregation directly from their `ObjectParser`. + +commit a5e82d7fd620ba04f41f989283c6b73d2d4095f1 +Author: Ross Wolf <31489089+rw-access@users.noreply.github.com> +Date: Wed Mar 4 10:09:40 2020 -0700 + + EQL: Add explicit 'any where ...' handling (#52526) + +commit f68917160e8ce301511dca816e897db660f769bb +Author: Tim Brooks +Date: Wed Feb 26 15:03:40 2020 -0700 + + Fix RemoteConnectionManager size() method (#52823) + + Currently the remote connection manager will delegate the size() call to + the underlying cluster connection manager. This introduces the + possibility that call will return 1 before the nodeConnection method has + been triggered to add the connection to the remote connection list. This + can cause issues, as the ensureConnected method checks the connection + managers size and executes synchronously if the size is > 0. This leads + to a potential cluster not connected exception while we are still + waiting for the connection opened callback to be triggered. + + This commit fixes this issue by using the remote connection manager's + size to report the connection manager's size. + + Fixes #52029. + +commit 8ab74fea581dabaebbe516d1aea6a342516c9de2 +Author: Yannick Welsch +Date: Wed Mar 4 17:39:09 2020 +0100 + + [7.x] Add 7.6.2 as version (#53114) + +commit f08ed1f69a99d2dd6038b35d3271d1bba77ea332 +Author: Jake Landis +Date: Wed Mar 4 10:38:07 2020 -0600 + + [7.x] add 6.8.8 as version (#53021) + +commit dfebbbf862fe03327501d0f3881d2887b27dfbcc +Author: Alan Woodward +Date: Wed Mar 4 15:37:07 2020 +0000 + + BoolQueryBuilder uses ObjectParser (#52880) + + This commit removes the hand-rolled x-content parsing logic from BoolQueryBuilder + and instead uses an ObjectParser to handle parsing. It also removes the long-deprecated + (since version 6) disable_coord parameter. + +commit 812d981d4d064aabd81e6d92182263cd7a585dd9 +Author: Mark Vieira +Date: Wed Mar 4 07:36:20 2020 -0800 + + Update to reflect version of SLES used in CI + + Signed-off-by: Mark Vieira + +commit 609c61f75c71a230637e53f2418e7f5ff3d2f5a3 +Author: Nik Everett +Date: Wed Mar 4 10:29:11 2020 -0500 + + Formalize usage stats for analytics (backport of #52966) (#53077) + + This moves the usage statistics gathering from the `AnalyticsPlugin` + into an `AnalyicsUsage`, removing the static state. It also checks the + license level when parsing all analytics aggregations. This is how we + were checking them before but we did it in an easy to forget way. This + way is slightly simpler, I think. + +commit 801e50203e6a6e8ab0f3402c3a868343a6c0bd5a +Author: James Rodewig +Date: Wed Mar 4 10:25:19 2020 -0500 + + [DOCS] Add missing doc type to EQL search results + +commit 3fa5395ac8d5ae21421f8b2d1a2dc276cf5c2c4d +Author: Martijn van Groningen +Date: Wed Mar 4 16:17:28 2020 +0100 + + Use correct issue number: #52453 + +commit 2e325e24cb28a1ff5152382c39cbea3198472240 +Author: Martijn van Groningen +Date: Wed Mar 4 16:07:48 2020 +0100 + + Mute testMonitorClusterHealth test (#53109) + + Relates to #36782 + +commit d1e7951e0081e7b4c199c3dd569d9f77dee2c638 +Author: Yannick Welsch +Date: Wed Mar 4 15:46:20 2020 +0100 + + [DOCS] Add 7.6.1. release notes (#52874) + + Adds the release notes for 7.6.1. + +commit b77f6746d19cf0810518885146af644c0caf66cb +Author: Martijn van Groningen +Date: Mon Feb 24 09:04:49 2020 +0100 + + unmute watcher single node test case + + relates to #36782 + +commit e3d3c3400cfd46429ce60640a748d4b2af4eba0a +Author: James Rodewig +Date: Wed Mar 4 09:16:27 2020 -0500 + + [DOCS] Update EQL default event category and timestamp values (#53102) + + Updates the documented default `event_category_field` and `timestamp_field` + values for the EQL search API. Also updates related guidance in the + EQL requirement docs. + + Relates to #53073. + +commit 0c4bf64095d8cd9850170fc51aefc20f553e57bc +Author: James Rodewig +Date: Wed Mar 4 08:42:37 2020 -0500 + + [DOCS] Fix several Asciidoctor double arrow replacements (#52827) + + Per the [Asciidoctor docs][0], Asciidoctor replaces the following + syntax with double arrows in the rendered HTML: + + * => renders as ⇒ + * <= renders as ⇐ + + This escapes several unintended replacements, such as in the Painless + docs. + + Where appropriate, it also replaces some double arrow instances with + single arrows for consistency. + + [0]: https://asciidoctor.org/docs/user-manual/#replacements + +commit 2b59f8ac34855abff686dde15456828771a11655 +Author: James Rodewig +Date: Wed Mar 4 08:22:40 2020 -0500 + + [DOCS] Correct `hits.total.relation` response parm def (#52847) + + Fixes a partially completed definition for the `hits.total.relation` + response parameter in the search API docs. + +commit b47bffba24cb3bc8533ddc53f1c88bfb6fb5a159 +Author: Aleksandr Maus +Date: Wed Mar 4 08:02:38 2020 -0500 + + EQL: consistent naming for event type vs event category (#53073) (#53090) + + Related to https://github.com/elastic/elasticsearch/issues/52941 + +commit e180e2738aee9279538ad5a1d6f573e7ce3ea1e9 +Author: Marios Trivyzas +Date: Wed Mar 4 11:47:23 2020 +0100 + + SQL: [Tests] Add tests for optimization of aliased expressions (#53048) + + Add a unit test to verify that the optimization of expression + (e.g. COALESCE) is applied to all instances of the expression: + SELECT, WHERE, GROUP BY and HAVING. + + Relates to #35270 + + (cherry picked from commit 2ceedc7f2019fad92cd86679af1a9c6fa594aa8d) + +commit 1d5c84270033bc1b919019f9ac61ebf1c6dc02f8 +Author: Marios Trivyzas +Date: Wed Mar 4 10:09:48 2020 +0100 + + SQL: Fix column size for IP data type (#53056) + + Set size/displaySize to 45 which is the maximum string for + an IP (v6), since IPs are returned as strings. + + Fixes: #52762 + + (cherry picked from commit 815f01747a4d54a274ca248af6fc08e5ea0728c1) + +commit 4b528d97adf2824a2eefcb683fa6fe7f8bf511b2 +Author: Mark Vieira +Date: Tue Mar 3 14:40:55 2020 -0800 + + Consolidate duplication of BWC testing task setup in script plugin (#53079) + + + (cherry picked from commit 33fc8e7ebfac8d47a5f9f026b3836bb47bea141a) + +commit 3fcf598b92ef0c4f44e0019a4a5fcb4868cffbcf +Author: Zachary Tong +Date: Tue Mar 3 17:07:40 2020 -0500 + + Reduce deprecation log noise from DateIntervalWrapper (#52655) + + Converts the deprecations to `deprecatedAndMaybeLog` to reduce the + number of times we log deprecations, since some of these could be called + at a high frequency (due to unconverted queries, aggs, etc) + +commit c610e0893db3e713bb9eb7d5d1335b9053681638 +Author: Jay Modi +Date: Tue Mar 3 14:11:37 2020 -0700 + + Introduce system index APIs for Kibana (#53035) + + This commit introduces a module for Kibana that exposes REST APIs that + will be used by Kibana for access to its system indices. These APIs are wrapped + versions of the existing REST endpoints. A new setting is also introduced since + the Kibana system indices' names are allowed to be changed by a user in case + multiple instances of Kibana use the same instance of Elasticsearch. + + Additionally, the ThreadContext has been extended to indicate that the use of + system indices may be allowed in a request. This will be built upon in the future + for the protection of system indices. + + Backport of #52385 + +commit 7339427af532bcee0aadcf97383b62b1b7038362 +Author: Nik Everett +Date: Tue Mar 3 15:27:49 2020 -0500 + + Remove some deprecation warnings parsing aggs (backport of #53026) (#53072) + + With #50871 aggrgations should now be parsed directly by an + `ObjectParser` or `ConstructingObjectParser` without the need for the + ceremonial `parse` method. This removes 10 of those `parse` methods and + parses the aggregation directly from their `ObjectParser`. + +commit 892f0d5848fcd9a10b851934316b8e139d7da127 +Author: Lisa Cawley +Date: Tue Mar 3 10:28:54 2020 -0800 + + [DOCS] Adds link in datafeed indices_options (#53067) + +commit bb4bcbc54c16f341bd872acb1b0873e5c9482cec +Author: Lisa Cawley +Date: Tue Mar 3 10:13:21 2020 -0800 + + [DOCS] Adds ignore_throttled to index conventions (#53063) + +commit cf87724ff6e4cc284bb86803c5d411d2dbf6d2fc +Author: James Rodewig +Date: Tue Mar 3 13:22:52 2020 -0500 + + [DOCS] Reformat `stop` token filter (#53059) + + Makes the following changes to the `stop` token filter docs: + + * Updates description + * Adds a link to the related Lucene filter + * Adds detailed analyze snippet + * Updates custom analyzer and custom filter snippets + * Adds a list of predefined stop words by language + + Co-authored-by: ScottieL <36999642+ScottieL@users.noreply.github.com> + +commit 9ad9ad7a6ba9695a68aad4eb6702cb7b8e8b70d5 +Author: Andrei Stefan +Date: Tue Mar 3 19:37:37 2020 +0200 + + SQL: update SqlNodeSubclassTests list of min-two-parameters functions list (#53045) (#53058) + + (cherry picked from commit c741e49d9f5e7b78c1a78e1af97eb19354fe6864) + +commit d3a8ac66c6e48d15bf61a2544971d30fbdb0c34e +Author: William Brafford +Date: Tue Mar 3 12:01:54 2020 -0500 + + Handle special chars in JAVA_HOME in elasticsearch-service.bat (#52676) (#53057) + + * Handle special chars in JAVA_HOME in elasticsearch-service.bat (#52676) + + * Test case for windows service where JAVA_HOME path contains spaces (#53028) + + Co-authored-by: Muhammad Shaheer Akram <41253927+shaheerakr@users.noreply.github.com> + +commit 6cece3a7099733baae2ff4bb7be31ffba067ea93 +Author: István Zoltán Szabó +Date: Tue Mar 3 16:25:24 2020 +0100 + + [DOCS] Adds response body documentation to GET inference API (#53050) + +commit 8a05b670ca1ae160319de6cf60cf0e95581ea691 +Author: Luca Cavanna +Date: Tue Mar 3 16:08:10 2020 +0100 + + Address MinAndMax generics warnings (#52642) + + `MinAndMax` encapsulates min and max values for a shard. It uses generics to make sure that the values are of the same type and are also comparable. Though there are warnings whenever this class is currently used, which are addressed with this commit. + + Relates to #49092 + +commit cb868d2f5ec6aad4b8fb7b670e7b1426371202fa +Author: Adrien Grand +Date: Tue Mar 3 16:01:47 2020 +0100 + + Introduce a `constant_keyword` field. (#49713) (#53024) + + This field is a specialization of the `keyword` field for the case when all + documents have the same value. It typically performs more efficiently than + keywords at query time by figuring out whether all or none of the documents + match at rewrite time, like `term` queries on `_index`. + + The name is up for discussion. I liked including `keyword` in it, so that we + still have room for a `singleton_numeric` in the future. However I'm unsure + whether to call it `singleton`, `constant` or something else, any opinions? + + For this field there is a choice between + 1. accepting values in `_source` when they are equal to the value configured + in mappings, but rejecting mapping updates + 2. rejecting values in `_source` but then allowing updates to the value that + is configured in the mapping + This commit implements option 1, so that it is possible to reindex from/to an + index that has the field mapped as a keyword with no changes to the source. + + Backport of #49713 + +commit bcb68c860c0c8e332acd9dcacf59dc019087a886 +Author: James Rodewig +Date: Tue Mar 3 06:59:17 2020 -0500 + + [DOCS] Reorganize EQL requirements page + +commit 70814daa86a9c9a9de75e1cecfbea11265cebdc1 +Author: Yang Wang +Date: Tue Mar 3 22:29:54 2020 +1100 + + Allow _rollup_search with read privilege (#52043) (#53047) + + Currently _rollup_search requires manage privilege to access. It should really be + a read only operation. This PR changes the requirement to be read indices privilege. + + Resolves: #50245 + +commit 3759063d342bfdae83095b4acf6e07176235cbd2 +Author: Alan Woodward +Date: Tue Mar 3 09:50:32 2020 +0000 + + Allow specifying an exclusive set of fields on ObjectParser (#52893) + + ObjectParser allows you to declare a set of required fields, such that at least one + of the set must appear in an xcontent object for it to be valid. This commit adds + the similar concept of a set of exclusive fields, such that at most one of the set + must be present. It also enables required fields on ConstructingObjectParser, and + re-implements PercolateQueryBuilder.fromXContent() to use object parsing as + an example of how this works. + +commit 510db25dd0542305f84278fc082db439e5ce57b4 +Author: Martijn van Groningen +Date: Tue Mar 3 11:01:57 2020 +0100 + + Simplify watcher indexing listener.(#53046) + + Backport: #52627 + + Add watcher to trigger server after index operation has succeeded, + instead of adding a watch to trigger service before + the actual index operation has performed on the shard level. + + This logic is simpler to reason about in the case that a failure + does occur during the execution of an index operation on + the shard level. + + Relates to #52453, but I think doesn't fix it, but makes it easier + to debug. + +commit 844f350774fc0077d1705a2ad00a06168907881a +Author: Hendrik Muhs +Date: Tue Mar 3 10:30:55 2020 +0100 + + [Transform] restructure transform yaml tests (#52956) + + restructure transform yaml tests to run cleanup in teardown phase + + relates #52428 + +commit d9258e210ee4d47d9e03b3923bcfed5e4fae5d76 +Author: Hendrik Muhs +Date: Tue Mar 3 10:01:04 2020 +0100 + + [Transform] fix sporadic race condition in TransformUsageIT (#52946) + + relax the test for trigger count + + fixes #52931 + +commit a154f9c657d18d841359859cb2850eb66bc1f333 +Author: Jason Tedor +Date: Mon Mar 2 23:27:33 2020 -0500 + + Early return if no global checkpoint listeners (#53036) + + When notifying global checkpoint listeners, we have an opportunity to + early return if there are not any registered listeners. This is + important since it saves some allocations, and also saves forking some + empty work to another thread. This commit adds an early return from + notifying listeners if there are not any registered. + +commit 5cffa14f45d6657c00bd572b9f0c1b3911d8ba5d +Author: James Rodewig +Date: Mon Mar 2 16:07:42 2020 -0500 + + [DOCS] Fix typo in EQL docs + +commit 712e0c05cd8cf9b2da47df230b242c91630f7eb4 +Author: Costin Leau +Date: Mon Mar 2 22:40:05 2020 +0200 + + EQL: Add implicit ordering on timestamp (#53004) + + QL: Move Sort base class from SQL to QL + (cherry picked from commit 798015b7bbd565e9c4222724614baeb432c7c2b3) + +commit f8396e8d154ae847469999c256bd725361de036d +Author: Mark Vieira +Date: Mon Mar 2 09:20:53 2020 -0800 + + Mute RunDataFrameAnalyticsIT.testStopOutlierDetectionWithEnoughDocumentsToScroll + + Signed-off-by: Mark Vieira + +commit 5b5e92c71dcb91dbcaadacd226969d6235a1cbb2 +Author: Mark Vieira +Date: Mon Mar 2 09:15:39 2020 -0800 + + Mute NodeSubclassTests.testReplaceChildren + + Signed-off-by: Mark Vieira + +commit 5a293b29a19a5b63fcb9f7d0879314582efac3bd +Author: Lisa Cawley +Date: Mon Mar 2 09:05:20 2020 -0800 + + [DOCS] Add missing anchors in cron expressions (#52960) + +commit ad2f630795a0b55fe9f76df52527408fea1e2780 +Author: Orhan Toy +Date: Mon Mar 2 17:41:22 2020 +0100 + + [DOCS] Fix formatting of simulate ingest pipeline API docs (#52754) + +commit a267849def522848370b6788ce2551e276a240d2 +Author: István Zoltán Szabó +Date: Mon Mar 2 10:48:09 2020 +0100 + + [DOCS] Adds transform-settings page and its subpage to redirects (#52944) + + Co-authored-by: Lisa Cawley + +commit 4fbe1b0550399e262ec8170674cf36eeafd2032b +Author: Lisa Cawley +Date: Mon Mar 2 07:28:55 2020 -0800 + + [DOCS] Adds cat anomaly detectors API (#52866) (#52970) + +commit a328a8eaf13ceaaaec2f86eb6173187f02c6c2b6 +Author: Hendrik Muhs +Date: Mon Mar 2 16:10:57 2020 +0100 + + [7.x][Transform] implement node.transform to control where to… (#52998) + + implement transform node attributes to disable transform on certain nodes and + test which nodes are allowed to do remote connections + + closes #52200 + closes #50033 + closes #48734 + + backport #52712 + +commit db6402991920918160528fe0152441d1edd455fe +Author: James Rodewig +Date: Mon Mar 2 10:08:03 2020 -0500 + + [7.x] [DOCS] Add parameter examples to EQL search tutorial (#52953) + + Makes the following updates to the EQL search tutorial: + + * Adds an API response to the basic tutorial + * Adds an example using the `event_type_field` parm + * Adds an example using the `timestamp_field`parm + * Adds an example using the `query` parm + * Updates example dataset to support more EQL query variety + +commit 89ed857c79d4b7a4fe105835ad03fcfd9d1bdf8a +Author: Aleksandr Maus +Date: Mon Mar 2 09:26:23 2020 -0500 + + EQL: Change request parameter query to filter and rule to query (#52971) (#53006) + + Related to https://github.com/elastic/elasticsearch/issues/52911 + +commit d336faa0b092dc2ff331982f19800a2771250403 +Author: James Rodewig +Date: Mon Mar 2 07:47:38 2020 -0500 + + [DOCS] Reformat trim token filter docs (#51649) + + Makes the following changes to the `trim` token filter docs: + + * Updates description + * Adds a link to the related Lucene filter + * Adds tip about removing whitespace using tokenizers + * Adds detailed analyze snippets + * Adds custom analyzer snippet + +commit f5bccad8474176c5ece100f7f8050e0efe9d8725 +Author: James Rodewig +Date: Mon Mar 2 07:37:48 2020 -0500 + + [DOCS] Correct guidance for `index_options` mapping parm (#52899) + + Adds a warning admonition stating that the `index_options` mapping + parameter is intended only for `text` fields. + + Removes an outdated statement regarding default values for numeric + and other datatypes. + +commit 7eb4c07f1f18f6947d30d8f216a932394165835e +Author: rhymes +Date: Mon Mar 2 13:22:27 2020 +0100 + + [DOCS] Fix typo in index and search analysis docs (#52988) + +commit 6fecc1db84385721d0d45d8535b2175b799682fe +Author: Andrei Stefan +Date: Mon Mar 2 14:04:49 2020 +0200 + + Issue a different error message in case an index doesn't have a mapping (#52967) (#53003) + + (cherry picked from commit a0bd83a0579cf196a1d727de2a46b3b101d5a73b) + +commit 69383acecfe31172155abd4dd9f858ae1546efc3 +Author: Andrei Stefan +Date: Mon Mar 2 11:26:50 2020 +0200 + + Define list of Nodes that have minimum two children in tests (#52957) (#52994) + + (cherry picked from commit c1e43e694f02edf3e197abbab7c21008c022b516) + +commit 49f41d127bcad5c21383f7d9c37099995aa2be3b +Author: Hendrik Muhs +Date: Mon Mar 2 08:02:26 2020 +0100 + + [Transform] fix NPE in derive stats if shouldStopAtNextCheckpo… (#52940) + + fixes a NPE in _stats in case shouldStopAtNextCheckpoint is set. + +commit d102158e6f0b4e47441aa34b6de381d2c110573e +Author: Martijn van Groningen +Date: Mon Mar 2 07:16:34 2020 +0100 + + Improve closing mock webserver when failed to start (#52943) + + Fix NPE when closing a webserver that hasn't started correctly. + + This can happen when ssl context isn't initialized. The server instance is then never set, + which causes an NPE that masks the actual failure. + + Example stacktrace that would mask an actual failure: + + ``` + java.lang.NullPointerException + at org.elasticsearch.test.http.MockWebServer.close(MockWebServer.java:271) + at org.elasticsearch.xpack.watcher.test.integration.HttpSecretsIntegrationTests.cleanup(HttpSecretsIntegrationTests.java:70) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:498) + ``` + +commit 712191f2af821d42704cc49fda56c499de890fa8 +Author: Yang Wang +Date: Mon Mar 2 11:56:42 2020 +1100 + + [DOCS] Fix typo and ensure asterisks render properly (#52991) (#52992) + + Fix a typo and rendering issue for doc test example. + +commit 210aab09351805edca5821d7dc9d44854b05e4bd +Author: Stuart Tettemer +Date: Sat Feb 29 09:38:46 2020 -0700 + + Settings: AffixSettings as validator dependencies (#52973) (#52982) + + Allow AffixSetting as validator dependencies. If a validator + specifies AffixSettings as a dependency, then `validate(T, Map)` + will have the concrete setting in a map. + + Backport of: #52973, 1e0ba70 + Fixes: #52933 + +commit e6755afeebf1ba8286154af23f1f7a59b6a5d0c9 +Author: Nhat Nguyen +Date: Sat Feb 29 09:29:16 2020 -0500 + + Upgrade to Lucene 8.5.0-snapshot-c4475920b08 (#52950) (#52977) + + To give LUCENE-9228 more CI cycles + +commit 85b4e4509352b7462baddbad6e5b4ac88aba44b0 +Author: Dimitris Athanasiou +Date: Sat Feb 29 13:03:40 2020 +0200 + + [7.x]ML] Parse and report memory usage for DF Analytics (#52778) (#52980) + + Adds reporting of memory usage for data frame analytics jobs. + This commit introduces a new index pattern `.ml-stats-*` whose + first concrete index will be `.ml-stats-000001`. This index serves + to store instrumentation information for those jobs. + + Backport of #52778 and #52958 + +commit d9463983af9fb66acb8283cf33609f301e38c95b +Author: Mark Vieira +Date: Fri Feb 28 15:08:13 2020 -0800 + + Enable parallel builds by default (#52972) + +commit b1be7dcd2d59a1020d5ac1b1c7562c7d6bb41b20 +Author: Rory Hunter +Date: Fri Feb 28 21:26:53 2020 +0000 + + Document how to change GC logging behaviour (#52879) + + Closes #43990. Describe how to change the default GC settings without changing + the default `jvm.options`. Give examples using `jvm.options.d`, and + `ES_JAVA_OPTS` with Docker. + +commit 1cd0eee7232e7fc6ca6841996dca014d713cfc8c +Author: Jay Modi +Date: Fri Feb 28 13:56:28 2020 -0700 + + Remove TODO in IndexNameExpressionResolver (#52969) + + This commit removes a TODO in the IndexNameExpressionResolver that + indicated the API should use a Set instead of a List. However, this + TODO was not completely correct since the ordering of arguments matters + due to negations when evaluating wildcards and since we also allow + a list of patterns like `*,-foo,*`, which would have a different + meaning even when using a Set with insertion ordering. + + Relates #52788 + Backport of #52963 + +commit 39de995740e0af5ba1f676cdda4b6b40764cdc96 +Author: Rory Hunter +Date: Fri Feb 28 20:54:47 2020 +0000 + + Exclude generated source from benchmarks formatting (#52968) + + IDEs can sometimes run annotation processors that leave files in + `src/main/generated/**/*.java`, causing Spotless to complain. Even + though this path ought not to exist, exclude it anyway in order to avoid + spurious failures. + +commit 331d4bb0afed944f25b719b0d4dbff2e0ccbc4fd +Author: Adrien Grand +Date: Fri Feb 28 18:45:46 2020 +0100 + + HybridDirectory should mmap postings. (#52641) (#52873) + + Since version 8.4, `MMapDirectory` has an optimization to read long[] + arrays directly in little endian order, which postings leverage. So it'd + be more efficient to open postings with `MMapDirectory`. + + I refactored a bit the existing logic to better explain why every listed + file extension is open with `mmap`. + +commit 090bdf69c0f89117f2e4d823e604e2700abc8f5a +Author: Luca Cavanna +Date: Fri Feb 28 16:13:17 2020 +0100 + + Mute NodeSubclassTests#testReplaceChildren (#52952) + + Relates #52951 + +commit dd44376d27f676fc6f1cceb1548e0f0ec0450b70 +Author: Dan Hermann +Date: Fri Feb 28 08:57:35 2020 -0600 + + [7.x] Send the fields param in body instead of URL params (#52948) + +commit d863c510da1c74da20aec554e8f19fc0cf895b52 +Author: Rory Hunter +Date: Fri Feb 28 14:48:04 2020 +0000 + + Autoformat :qa:os and :benchmarks (#52816) + + Add `:qa:os` and `:benchmarks` to the list of automatically formatted + projects, and apply some manual fix-ups to polish it up. + + In particular, I noticed that `Files.write(...)` when passed a list will + automaticaly apply a UTF-8 encoding and write a newline after each line, + making it easier to use than FileUtils.append. It's even available from + 1.8. + + Also, in the Allocators class, a number of methods declared thrown exceptions that IntelliJ reported were never thrown, and as far as I could see this is true, so I removed the exceptions. + +commit c3a167830fd7c3b60a6abf8513ba261d161c6854 +Author: Andrei Stefan +Date: Fri Feb 28 14:04:10 2020 +0200 + + SQL: refactor In predicate moving it to QL project (#52870) (#52938) + + * Move In, InPipe and InProcessor out of SQL to the common QL project. + * Move tests classes to the QL project. + * Create SQL dedicated In class to handle SQL specific data types. + * Update SQL classes to use the InPipe and InProcessor QL classes. + * Extract common Foldables methods in QL project. + * Be more explicit when folding and converting a foldable value, by + removing most of the code inside Foldables class. + + (cherry picked from commit 7425042f86f66df8c207c5e96f9b9848bda2b4c3) + +commit a67408590304a08aeec93726c381d4efd3fea22a +Author: Costin Leau +Date: Fri Feb 28 13:45:21 2020 +0200 + + EQL: Disable field extraction for returned events (#52884) + + Return the whole source of matching events + + (cherry picked from commit 79ca586ab1d89d645fb58142b82202f14ce5d361) + +commit 6aa9aaa2c62bd1e052b3146e06e9db4ff2fd2c30 +Author: Martijn van Groningen +Date: Fri Feb 28 10:35:04 2020 +0100 + + Add validation for dynamic templates (#52890) + + Backport of #51233 to the seven dot x branch. + + Tries to load a `Mapper` instance for the mapping snippet of a dynamic template. + This should catch things like using an analyzer that is undefined or mapping attributes that are unused. + + This is best effort: + * If `{{name}}` placeholder is used in the mapping snippet then validation is skipped. + * If `match_mapping_type` is not specified then validation is performed for all mapping types. + If parsing succeeds with a single mapping type then this the dynamic mapping is considered valid. + + If is detected that a dynamic template mapping snippet is invalid at mapping update time then the mapping update is failed for indices created on 8.0.0-alpha1 and later. For indices created on prior version a deprecation warning is omitted instead. In 7.x clusters the mapping update will never fail in case of an invalid dynamic template mapping snippet and a deprecation warning will always be omitted. + + Closes #17411 + Closes #24419 + + Co-authored-by: Adrien Grand + +commit c642a97255bc206b6970852b6c6d51a37378d248 +Author: Mark Vieira +Date: Thu Feb 27 19:18:53 2020 -0800 + + Support parallel building of Docker images (#52920) + +commit 82553524af75ff4ccde59edb225c8587053cc8c9 +Author: Yang Wang +Date: Fri Feb 28 10:53:52 2020 +1100 + + Respect runas realm for ApiKey security operations (#52178) (#52932) + + When user A runs as user B and performs any API key related operations, + user B's realm should always be used to associate with the API key. + Currently user A's realm is used when getting or invalidating API keys + and owner=true. The PR is to fix this bug. + + resolves: #51975 + +commit 866b08716c4bb948f6c78c25bd52190d76e70ca7 +Author: Nik Everett +Date: Thu Feb 27 18:30:37 2020 -0500 + + Fix test for top_metrics (#52927) + + I added the wrong skips and the wrong error message. Ooops. + +commit 407101c39b7936c11ddc8470c11c373e0c134b52 +Author: Nik Everett +Date: Thu Feb 27 17:50:55 2020 -0500 + + Clean and document sorting with partialy built buckets (backport of #52769) (#52925) + + The `terms` aggregation can be sortd by the results of its + sub-aggregations. Because it uses that sorting for filtering to the + top-n it tries not to construct all of the buckets for the child + aggregations. This has its own interesting problem around reduction, but + they aren't super relevant to this change. This change moves that + optimization from the `TermsAggregator` and into the aggregators being + sorted on. This should make it more clear what is going on and it + unifies this optimization with validating the sort. + + Finally, this should enable some minor optimizations to save a few + comparisons when sorting multi-valued buckets. I'll get those in a + follow up because they are now *fairly* obvious. They probably won't be + a huge performance improvement, but it'll be nice anyway. + +commit a47e4047325c61b6433bb7fd2b72aac07482359c +Author: Lee Hinman +Date: Thu Feb 27 15:16:26 2020 -0700 + + Mute GoogleCloudStorageBlobStoreRepositoryTests (#52926) + + These intermittently fail due to an assertion triggered by a JDK bug. + + Relates to #52906 + +commit 1d1956ee93d30032f3e9dce318991fffff6fbdca +Author: Nik Everett +Date: Thu Feb 27 16:12:52 2020 -0500 + + Add size support to `top_metrics` (backport of #52662) (#52914) + + This adds support for returning the top "n" metrics instead of just the + very top. + + Relates to #51813 + +commit d568345f1af8e26d2ecab21e545fd8737f1538fa +Author: Jason Tedor +Date: Thu Feb 27 15:25:53 2020 -0500 + + Fix roles parsing in client nodes sniffer (#52888) + + We mades roles pluggable, but never updated the client to account for + this. This means that when speaking to a modern cluster, application + logs are spammed with warning messages around unrecognized roles. This + commit addresses this by accounting for the fact that roles can extend + beyond master/data/ingest now. + +commit 19a6c5d9803b2e8c48460273727f75bf019181da +Author: Benjamin Trent +Date: Thu Feb 27 14:05:28 2020 -0500 + + [7.x] [ML][Inference] Add support for multi-value leaves to the tree model (#52531) (#52901) + + * [ML][Inference] Add support for multi-value leaves to the tree model (#52531) + + This adds support for multi-value leaves. This is a prerequisite for multi-class boosted tree classification. + +commit 710a9ead69fac9901ffd0753dfdb98dd1536f18e +Author: Jake Landis +Date: Thu Feb 27 12:45:02 2020 -0600 + + [7.x] Updates to TESTING.asciidoc for REST testing (#52862) (#52898) + + Update documentation for: + * restResources config (related #52114) + * call out YAML vs. Java based Rest tests + * update example to use newer syntax + * update example to target a test that is not skipped + * provide example for bwcRest test (related #52383) + +commit 1c29140f2440e75380800e48559a794718fca91b +Author: Jake Landis +Date: Thu Feb 27 12:44:00 2020 -0600 + + [7.x] Fix incremental build support copying REST api and tests (#52896) + + A recent PR #52114 introduced two new tasks to copy the REST api and tests. + A couple bugs were found in that initial PR that prevents the incremental + build from working as expected. + + The pattern match of empty string is equivalent to match all and it was coded + as match none. Fixed with explicit checks against empty patterns. + + The fileCollection.plus return value was ignored. Fixed by changing how the + input's fileTree is constructed. + + If a project has an src/test/resources directory, and tests are being copied + without a rest-api-spec/test directory could result no-op. Masked by the other + bugs and fixed by minor changes to logic to determine if a project has tests. + +commit eac38e9847ae3cc89b89acdffade306fe78dda71 +Author: Benjamin Trent +Date: Thu Feb 27 13:43:25 2020 -0500 + + [ML] Add indices_options to datafeed config and update (#52793) (#52905) + + This adds a new configurable field called `indices_options`. This allows users to create or update the indices_options used when a datafeed reads from an index. + + This is necessary for the following use cases: + - Reading from frozen indices + - Allowing certain indices in multiple index patterns to not exist yet + + These index options are available on datafeed creation and update. Users may specify them as URL parameters or within the configuration object. + + closes https://github.com/elastic/elasticsearch/issues/48056 + +commit f46b370e7a74a61526839646ba0775f1ddf0c32b +Author: Mark Vieira +Date: Thu Feb 27 09:53:33 2020 -0800 + + Fix cacheability of repository-hdfs integ tests (#52858) + +commit 14f847cc8f4bb35f2c31454ac933f257d5a78c1a +Author: Nattachai Suteerapongpan +Date: Thu Feb 27 23:30:03 2020 +0700 + + [DOCS] Fix typo in task management API docs (#52881) + +commit e139d70abeeea46987cd15cf0657aa6da493a870 +Author: Lee Hinman +Date: Thu Feb 27 09:28:27 2020 -0700 + + Remove TODO in MaxSizeCondition (#52854) + + Similar to what we did in #52794, this removes the TODO. + + Relates again to #52505 + +commit 09fe4b42db65f2060443c4018c2a76041102f6ab +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Feb 27 17:19:58 2020 +0100 + + Disable ILM history in x-pack rest tests (#52868) + + The ILM history index can be delayed created from one test into the + next, which can cause issues for tests using `_all`. + + Closes #52209 + +commit 09d25ef8182b32efb7389313986bb2fcb61dad70 +Author: jureaky +Date: Fri Feb 28 01:07:35 2020 +0900 + + [DOCS] Fix typos in build.gradle comments (#52883) + +commit 68ba571f70abeec030b0b636fdef2401603e772a +Author: Josh Devins +Date: Thu Feb 27 16:04:24 2020 +0100 + + Adds recall@k metric to rank eval API (#52889) + + This change adds the recall@k metric and refactors precision@k to match + the new metric. + + Recall@k is an important metric to use for learning to rank (LTR) + use-cases. Candidate generation or first ranking phase ranking functions + are often optimized for high recall, in order to generate as many + relevant candidates in the top-k as possible for a second phase of + ranking. Adding this metric allows tuning that base query for LTR. + + See: https://github.com/elastic/elasticsearch/issues/51676 + Backports: https://github.com/elastic/elasticsearch/pull/52577 + +commit 3c8b46a8c1834c921d3e7199dcd357eca63c90c9 +Author: Dan Hermann +Date: Thu Feb 27 09:00:51 2020 -0600 + + [7.x] Handle errors when evaluating if conditions in processors (#52892) + +commit 8785f57dfe07ff204c0f6bdc9dd4c6ff75f3e4c9 +Author: István Zoltán Szabó +Date: Thu Feb 27 14:20:29 2020 +0100 + + [DOCS] Reformats cat DFA API docs. (#52885) + +commit d8bdf31110521733e5cfc3e7a2da2fa3450db2f3 +Author: David Kyle +Date: Thu Feb 27 12:38:13 2020 +0000 + + Revert "Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart" + + This reverts commit ad3a3b1af984bc051e7af01b50d1f4c78120e44d. + +commit 6e5e64559a6a0682548a744041ef3038bd7d956c +Author: David Kyle +Date: Thu Feb 27 11:58:28 2020 +0000 + + Unwrap cause from remote ActionTransportExceptions (#52842) (#52878) + + And log the cause + +commit 4a33352a948581b289e2241e3ecb708536e2954b +Author: István Zoltán Szabó +Date: Thu Feb 27 12:50:11 2020 +0100 + + [DOCS] Adds cat trained model API documentation (#52824) + +commit 280d59c724c00cd3f6182a3978c668fa869d4f85 +Author: hezhen Zhang +Date: Thu Feb 27 19:15:40 2020 +0800 + + Append index name for the source of the cluster put-mapping task (#52690) + + Add index name(s) into the source for the cluster state update done when putting mapping. + This ensures that the pending tasks API includes information on source indices. + +commit 52fa4653003965283a299bc1872726b242fc7dbc +Author: David Turner +Date: Thu Feb 27 10:01:24 2020 +0000 + + Cache completion stats between refreshes (#52872) + + Computing the stats for completion fields may involve a significant amount of + work since it walks every field of every segment looking for completion fields. + Innocuous-looking APIs like `GET _stats` or `GET _cluster/stats` do this for + every shard in the cluster. This repeated work is unnecessary since these stats + do not change between refreshes; in many indices they remain constant for a + long time. + + This commit introduces a cache for these stats which is invalidated on a + refresh, allowing most stats calls to bypass the work needed to compute them on + most shards. + + Closes #51915 + Backport of #51991 + +commit 40bc06f6ad91b7a31dc3135eab21c9611d545773 +Author: Costin Leau +Date: Thu Feb 27 11:16:26 2020 +0200 + + EQL: Hook engine to Elasticsearch (#52828) + + Add query execution and return actual results returned from + Elasticsearch inside the tests + + (cherry picked from commit 3e039282bf991af87604a6d4f8eada19d5e33842) + +commit 69b78f7f8abfc377f3a2500e27fdb06c67a71efc +Author: David Turner +Date: Thu Feb 27 08:51:17 2020 +0000 + + "Adding nodes" instructions only work on localhost (#52677) + + The introductory sections of the reference manual contains some simplified + instructions for adding a node to the cluster. Unfortunately they are a little + too simplified and only really work for clusters running on `localhost`. If you + try and follow these instructions for a distributed cluster then the new node + will, confusingly, auto-bootstrap itself into a distinct one-node cluster. + + Multiple nodes running on localhost is a valid config, of course, but we should + spell out that these instructions are really only for experimentation and that + it takes a bit more work to add nodes to a distributed cluster. This commit + does so. + + Also, the "important config" instructions for discovery say that you MUST set + `discovery.seed_hosts` whereas in fact it is fine to ignore this setting and + use a dynamic discovery mechanism instead. This commit weakens this statement + and links to the docs for dynamic discovery mechanisms. + + Finally, this section is also overloaded with some technical details that are + not important for this context and are adequately covered elsewhere, and + completely fails to note that the default discovery port is 9300. This commit + addresses this. + +commit 14c21aedd2eb9b98ca2c139c70f4c322f24ea750 +Author: Yang Wang +Date: Thu Feb 27 14:14:16 2020 +1100 + + Simplify ml license checking with XpackLicenseState internals (#52684) (#52863) + + This change removes TrainedModelConfig#isAvailableWithLicense method with calls to + XPackLicenseState#isAllowedByLicense. + + Please note there are subtle changes to the code logic. But they are the right changes: + * Instead of Platinum license, Enterprise license nows guarantees availability. + * No explicit check when the license requirement is basic. Since basic license is always available, this check is unnecessary. + * Trial license is always allowed. + +commit fa701e4c1f82ce1cb869feb182187e2ffba249fd +Author: Nhat Nguyen +Date: Tue Feb 18 08:50:36 2020 -0500 + + Fix testRetentionLeasesEstablishedWhenRelocatingPrimary (#52445) + + Replace the current assertion with a more robust assertion. + + Closes #52364 + +commit f5c4e925585ca911033c6739a36a809f70bf821c +Author: Yang Wang +Date: Thu Feb 27 13:04:19 2020 +1100 + + Refactor license checking (#52118) (#52859) + + Improve code resuse and readility. Add convenience checking method which + covers most use cases without having to pass many boolean arguments. + +commit bc9c3f0135e977ada93d41f474a6fe97d4dbe778 +Author: Mark Vieira +Date: Wed Feb 26 14:29:16 2020 -0800 + + Ignore test seed in third party test system property inputs (#52849) + +commit 814c275f35f688c15e93e4f49608dc134276614e +Author: Nhat Nguyen +Date: Tue Feb 25 15:44:23 2020 -0500 + + Add more assertions to testMaybeFlush (#52792) + + We aren't able to reproduce or figure out the reason that failed this test. + This commit adds more assertions so we can narrow the scope. + + Relates #52223 + +commit 0a15a6bfadd7349a9aa4859d432432453e5ba3bd +Author: Nhat Nguyen +Date: Thu Feb 20 14:29:48 2020 -0500 + + Fix testSeqNoCollision (#52588) + + Adjusts the assertion as we trim translog more eagerly since #52556. + + Relates #52556 + Closes #52148 + +commit 87e765609ed8dacb5c91a0b426a198f79c78a128 +Author: Nhat Nguyen +Date: Fri Feb 21 03:22:38 2020 -0500 + + Fix testResyncAfterPrimaryPromotion (#52615) + + Adjusts the assertion as we might eagerly clean up translog during resync since #52556 + + Relates #52556 + Closes #52598 + +commit 5aa612c2759b9f025e429a0075161c4271c054a8 +Author: Nhat Nguyen +Date: Tue Feb 18 08:50:16 2020 -0500 + + Fix testRestoreLocalHistoryFromTranslog (#52441) + + Asserts that no new operations are made into the translog since we re-opened the engine. + + Relates #51905 + Closes #52410 + +commit a92bf5ec6105176f97954d68593dfe288427276b +Author: Nhat Nguyen +Date: Wed Feb 12 11:06:11 2020 -0500 + + Fix IndexShardIT#testMaybeFlush (#52247) + + Since #51905, we use the local checkpoint of the safe commit to + calculate the number of uncommitted operations of a translog stats. If a + periodic flush triggered by afterWriteOperation completes before we sync + translog, then the last commit is not safe. We also need to sync + translog from Engine instead of the translog so that we can advance the + safe commit. + + Relates #51905 + Closes #52223 + +commit d7fe135d90f74cf00eab61525c77ae80b74a4029 +Author: Nhat Nguyen +Date: Wed Feb 12 11:08:48 2020 -0500 + + Fix testPrepareIndexForPeerRecovery (#52245) + + Since #51905, we skip translog recovery if the local checkpoint of the + safe commit equals to the global checkpoint. This change adjusts the + test not to create a new snapshot in that case. + + Closes #52221 + Relates #51905 + +commit 82ab1bc1ff40a2e7041413428bfda810507f94e7 +Author: Yannick Welsch +Date: Thu Feb 20 15:55:33 2020 +0100 + + Separate translog from index deletion conditions (#52556) + + Separates the translog from the index deletion conditions (allowing the translog to be cleaned + up more eagerly), and avoids taking the write lock on the translog if no clean-up is actually + necessary. + +commit db6b9c21c714411caf443ba23786818a15f5b6d7 +Author: Nhat Nguyen +Date: Mon Feb 10 08:26:01 2020 -0500 + + Use local checkpoint to calculate min translog gen for recovery (#51905) + + Today we use the translog_generation of the safe commit as the minimum + required translog generation for recovery. This approach has a + limitation, where we won't be able to clean up translog unless we flush. + Reopening an already recovered engine will create a new empty translog, + and we leave it there until we force flush. + + This commit removes the translog_generation commit tag and uses the + local checkpoint of the safe commit to calculate the minimum required + translog generation for recovery instead. + + Closes #49970 + +commit b4179a8814d3d72fbe347f2525ff3e4c0ca49345 +Author: Jake Landis +Date: Wed Feb 26 15:57:10 2020 -0600 + + [7.x] Refactor watcher tests (#52799) (#52844) + + This PR moves the majority of the Watcher REST tests under + the Watcher x-pack plugin. + + Specifically, moves the Watcher tests from: + x-pack/plugin/test + x-pack/qa/smoke-test-watcher + x-pack/qa/smoke-test-watcher-with-security + x-pack/qa/smoke-test-monitoring-with-watcher + + to: + x-pack/plugin/watcher/qa/rest (/test and /qa/smoke-test-watcher) + x-pack/plugin/watcher/qa/with-security + x-pack/plugin/watcher/qa/with-monitoring + + Additionally, this disables Watcher from the main + x-pack test cluster and consolidates the stop/start logic + for the tests listed. + + No changes to the tests (beyond moving them) are included. + + 3rd party tests and doc tests (which also touch Watcher) + are not included in the changes here. + +commit 305c8342ac6e02835ee0fd1af462fa14f2fb9991 +Author: Mark Vieira +Date: Wed Feb 26 13:55:49 2020 -0800 + + Fix caching of build-tools project tests (#52848) + + We embed the :reaper project jar in the build-tools jar so we can spawn + a reaper process at build runtime. Due to this, the jar technically + isn't part of the test runtime classpath, but for input snapshotting + purposes, we should be treating it as such. Instead, because it lives + in META-INF, Gradle treats it as a normal file, which in practice means + its hash changes on every build (timestamps, etc). + + This commit changes our input snapshotting strategy such that instead + we explicitly add the jar as an input to any test tasks using Gradle's + runtime classpath normalization strategy (ignore timestamps, jar entry + order, etc) and ignore the file in META-INF. This ensures that we can + properly cache test results for build-tools, why still ensuring that + changes to the :reaper project trigger reexecution of tests. + +commit 59638bc4059ba5b8c4d26adfa316e20341911d5e +Author: JaeGeun +Date: Thu Feb 27 04:41:31 2020 +0900 + + Add UTF-8 encoding to build.gradle in BuildSrc/. (#52674) + +commit 3ffd34617f1066215f01bcb81251137bbed741e2 +Author: Dan Hermann +Date: Wed Feb 26 13:26:26 2020 -0600 + + Switch to AtomicLong for ingestCurrent metric to prevent negative values (#52581) (#52834) + +commit 07ef8ccff4c3b32cfcc3d39984e673158c0dfd0a +Author: Jay Modi +Date: Wed Feb 26 11:46:29 2020 -0700 + + Allow dynamic updates for index.hidden setting (#52837) + + This commit changes the `index.hidden` setting from being final to a + dynamic setting. While the setting being final allows for easier + reasoning about an index, making this setting update-able has more + benefits in that we can upgrade existing indices to be hidden and it + will enable future features that would dynamically make indices hidden. + + Backport of #52772 + +commit bfaa4877571890cde54ef7d82d2ad14f8b46acbc +Author: Nik Everett +Date: Wed Feb 26 12:57:20 2020 -0500 + + Switch pipeline agg parsing to ContextParser (#52776) (#52832) + + We've pretty well settled on `ContextParser` for a generic interface to + `ObjectParser`-like-things. This switches the interface used for + building parsing pipeline aggregations to `ContextParser` which saves a + couple of little wrappers around `ObjectParser`. + +commit f5253d20f7f320ce69ae1bc29d6ab1dc0392a7ed +Author: James Rodewig +Date: Wed Feb 26 12:35:05 2020 -0500 + + [DOCS] Update term vectors snippet to prevent CI failure (#52819) + + Adds the `?refresh=wait_for` query argument to an index API snippet in + the term vectors API docs. + + This should ensure the document is indexed and available before a + subsequent term vectors API request executes. + + Fixes #52814. + +commit f7d71b593062239412bb911bdb1f9fcac478e836 +Author: Armin Braun +Date: Wed Feb 26 18:38:02 2020 +0100 + + Fix GCS Mock Range Downloads (#52804) (#52830) + + We were not correctly respecting the download range which lead + to the GCS SDK client closing the connection at times. + Also, fixes another instance of failing to drain the request fully before sending the response headers. + + Closes #51446 + +commit b788ec7157815fd2675cf20abbbbbdb297956512 +Author: Lisa Cawley +Date: Wed Feb 26 09:20:36 2020 -0800 + + [DOCS] Adds cat datafeeds API (#52738) + +commit be8d704e2b3f551ad9b33b35bd61d44dbda793b7 +Author: Tim Brooks +Date: Wed Feb 26 10:17:25 2020 -0700 + + Remove seeds depedency for remote cluster settings (#52829) + + Currently 3 remote cluster settings (ping interval, skip unavailable, + and compression) have a dependency on the seeds setting being + comfigured. With proxy mode, it is now possible that these settings the + seeds setting has not been configured. This commit removes this + dependency and adds new validation for these settings. + +commit 2d01c005ba61410a84af743b589d0132d55e1ca1 +Author: Ioannis Kakavas +Date: Wed Feb 26 17:03:45 2020 +0200 + + Update commons-collections test dependency to 3.2.2 (#52808) (#52817) + + This is only a test dependency but it trips scanners so upgrade to + 3.2.2 which doesn't suffer from the issues mentioned in i.e. + https://snyk.io/vuln/SNYK-JAVA-COMMONSCOLLECTIONS-472711 + +commit 1807f86751aad208f4a8f23c8c836eb7f1f492ed +Author: Adrien Grand +Date: Wed Feb 26 15:37:43 2020 +0100 + + Generalize how queries on `_index` are handled at rewrite time (#52815) + + Generalize how queries on `_index` are handled at rewrite time (#52486) + + Since this change refactors rewrites, I also took it as an opportunity to adrress #49254: instead of returning the same queries you would get on a keyword field when a field is unmapped, queries get rewritten to a MatchNoDocsQueryBuilder. + + This change exposed a couple bugs, like the fact that the percolator doesn't rewrite queries at query time, or that the significant_terms aggregation doesn't rewrite its inner filter, which I fixed. + + Closes #49254 + +commit ad3a3b1af984bc051e7af01b50d1f4c78120e44d +Author: David Kyle +Date: Wed Feb 26 14:30:21 2020 +0000 + + Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart + +commit 8d311297ca13db3485db9e63afdae067b832e590 +Author: Jake Landis +Date: Wed Feb 26 08:13:41 2020 -0600 + + [7.x] Smarter copying of the rest specs and tests (#52114) (#52798) + + * Smarter copying of the rest specs and tests (#52114) + + This PR addresses the unnecessary copying of the rest specs and allows + for better semantics for which specs and tests are copied. By default + the rest specs will get copied if the project applies + `elasticsearch.standalone-rest-test` or `esplugin` and the project + has rest tests or you configure the custom extension `restResources`. + + This PR also removes the need for dozens of places where the x-pack + specs were copied by supporting copying of the x-pack rest specs too. + + The plugin/task introduced here can also copy the rest tests to the + local project through a similar configuration. + + The new plugin/task allows a user to minimize the surface area of + which rest specs are copied. Per project can be configured to include + only a subset of the specs (or tests). Configuring a project to only + copy the specs when actually needed should help with build cache hit + rates since we can better define what is actually in use. + However, project level optimizations for build cache hit rates are + not included with this PR. + + Also, with this PR you can no longer use the includePackaged flag on + integTest task. + + The following items are included in this PR: + * new plugin: `elasticsearch.rest-resources` + * new tasks: CopyRestApiTask and CopyRestTestsTask - performs the copy + * new extension 'restResources' + ``` + restResources { + restApi { + includeCore 'foo' , 'bar' //will include the core specs that start with foo and bar + includeXpack 'baz' //will include x-pack specs that start with baz + } + restTests { + includeCore 'foo', 'bar' //will include the core tests that start with foo and bar + includeXpack 'baz' //will include the x-pack tests that start with baz + } + } + + ``` + +commit 2a6c3bea3f0a224da90478c4c8ec46c116608e3f +Author: Ioannis Kakavas +Date: Wed Feb 26 16:02:10 2020 +0200 + + Update oauth2-oidc-sdk to 7.0 (#52489) (#52806) + + Resolves: #48409 + Other changes: + https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect + -extensions/src/7.0.2/CHANGELOG.txt + +commit f0bc8abcd022bef6c2a265efbd2f6439a1633e70 +Author: Nhat Nguyen +Date: Wed Feb 26 08:13:31 2020 -0500 + + Fix translog stats on closed indices yaml test (#52800) + + We need to wait for no initializing shards before closing; otherwise, + we might fail to close some recovering replicas. + + Closes #52701 + +commit 9e38125464a291526e32866f8de9b4fd5d572a8c +Author: Luca Cavanna +Date: Wed Feb 26 13:58:20 2020 +0100 + + Clarify when shard iterators get sorted (#52810) + + Currently we have two ways to create a GroupShardsIterator: one that will resort the iterators based on their natural ordering, and another one that will leave them in their original order. This is currently done through two constructors, one that accepts a single argument which does the sorting, and another which accepts a second boolean argument to control whether sorting should happen or not. This second constructor is only called externally to disable the sorting. + + By introducing a specific method to create a sorted shard iterator we clarify and make it easier to track when we do sort and when we do not as the iterators are externally sorted. + +commit 304e1e69b8aadbbdb8bf5f027dd5bc382ab00f70 +Author: Bogdan Pintea +Date: Wed Feb 26 13:32:22 2020 +0100 + + remove references to the SQL API from ODBC config (#52765) + + Remove reference to an "SQL API" which could suggest that one needs to + treat this in a special way when configuring the ODBC driver. + + (cherry picked from commit 451c341e0193b542409e8891ec2a31e62529a5e7) + +commit a76ec765e55c1a8e83666ec9f5474b979be80e6e +Author: Alan Woodward +Date: Wed Feb 26 10:51:28 2020 +0000 + + Ensure that percolator sorting also works (#52758) + + Commit #52748 fixed a bug where percolate queries wrapped in a constant score + could report incorrect matches. This commit adds a test to check that it also fixes + the case where a percolate query is sorted by something other than score. + + Closes #52618 + +commit f57422bbfd676b72c17fee28f3c4350e74804bc1 +Author: István Zoltán Szabó +Date: Wed Feb 26 11:09:37 2020 +0100 + + [DOCS] Adds cat data frame analytics API (#52764) + + Co-authored-by: Lisa Cawley + +commit a73ad248e841d5afe4bbe179675d212b79c3e607 +Author: Jim Ferenczi +Date: Wed Feb 26 10:46:51 2020 +0100 + + Fix backport of #46731 (#52744) + + This change fixes the incomplete backport of #46731 in 7.x (as of 7.5). + We now check if `max_children` is set on the top level nested sort and fails with an + exception if it's not the case. + + Relates #46731 + Closes #52202 + +commit 37be695d5ca52d04c3fd08927c8b647b9fad7872 +Author: David Kyle +Date: Wed Feb 26 08:18:37 2020 +0000 + + [ML] Handle failed datafeed in MlDistributedFailureIT (#52631) (#52789) + +commit 05f1cd74a6b18ea8e248dc2fbf88c2b25077f877 +Author: Lisa Cawley +Date: Tue Feb 25 16:57:38 2020 -0800 + + [DOCS] Fixes monitoring links (#52790) + +commit 85e5e565101274add7d427fce8f74d4f39a7fa6d +Author: Florian Kelbert +Date: Wed Feb 26 00:49:58 2020 +0000 + + [DOCS] Fixes missing colon (#52797) + +commit d3c0a2f013de07ff3a0dbde01566cd24c04cc658 +Author: Sachin Frayne <32795683+sachinfrayne@users.noreply.github.com> +Date: Tue Feb 25 23:28:51 2020 +0000 + + Improve the error message when loading text fielddata. (#52753) + + Emphasize keyword over fielddata as the preferred way to use String fields for aggregations or sorting. + +commit 662f21fceaee87ae4db8d72145bf70d20810d778 +Author: Lee Hinman +Date: Tue Feb 25 15:46:50 2020 -0700 + + Remove TODO in MaxAgeCondition serialization (#52794) + + * Remove TODO in MaxAgeCondition serialization + + This removes the TODO with a message for any future readers regarding the code in question. + + Resolves #52505 + +commit 6669e53f08d386c8806b28ccc72d9baab607a121 +Author: Tim Brooks +Date: Tue Feb 25 15:38:35 2020 -0700 + + Do not lock on reads of XPackLicenseState (#52492) + + XPackLicenseState reads to necessary to validate a number of cluster + operations. This reads occasionally occur on transport threads which + should not be blocked. Currently we sychronize when reading. However, + this is unecessary as only a single piece of state is updateable. This + commit makes this state volatile and removes the locking. + +commit c8ef9649e28c968d8401b59eeb5349930fee1362 +Author: Tim Brooks +Date: Tue Feb 25 14:37:11 2020 -0700 + + Force execution of finish shard bulk request (#51957) (#52484) + + Currently the shard bulk request can be rejected by the write threadpool + after a mapping update. This introduces a scenario where the mapping + listener thread will attempt to finish the request and fsync. This + thread can potentially be a transport thread. This commit fixes this + issue by forcing the finish action to happen on the write threadpool. + + Fixes #51904. + +commit 848d3bc1534bf72c81f2b705932146409c097768 +Author: Nhat Nguyen +Date: Tue Feb 25 14:12:35 2020 -0500 + + Revert "Fix testKeepTranslogAfterGlobalCheckpoint" + + This reverts commit a88d54eb2d710003f7cb838ba3f342e2b7ae5031. + +commit 924f0bd243eff15ffdd5f755b944193b89ca9a0e +Author: Lisa Cawley +Date: Tue Feb 25 09:30:14 2020 -0800 + + [DOCS] Updates custom rules example (#52731) + +commit 528e77cd5e600b67df781dfd1fd897ebcaea86da +Author: Mark Vieira +Date: Tue Feb 25 09:22:45 2020 -0800 + + Update Docker on linux exclusion list (#52736) + + (cherry picked from commit 5cfbf4dac11225d9f48fc25919e29713dd53a279) + +commit a88d54eb2d710003f7cb838ba3f342e2b7ae5031 +Author: Nhat Nguyen +Date: Tue Feb 25 11:46:43 2020 -0500 + + Fix testKeepTranslogAfterGlobalCheckpoint + + Read the last synced global checkpoint after flushing as + we might advance it during committing. + + CI: https://gradle-enterprise.elastic.co/s/7o6qengg4gva2 + +commit 51c6aefa5597bc51251fc2a21ec97bf4b5f17583 +Author: Andrei Stefan +Date: Tue Feb 25 18:44:03 2020 +0200 + + SQL: Use calendar_interval of 1d for HISTOGRAMs with 1 DAY intervals (#52749) (#52771) + + (cherry picked from commit 556f5fa33be88570c4f8550cb8f784323d26a707) + +commit a8911802d39ddb547a21c270792dfb3e81b68c39 +Author: Costin Leau +Date: Tue Feb 25 17:43:26 2020 +0200 + + EQL: transform query AST into queryDSL (#52432) + + (cherry picked from commit 94cef29df259319dfe2a3bf92d3f1a42d7e45781) + +commit 638f3e4183bf3db991c62836ea4f51ba2cf2d6da +Author: Alan Woodward +Date: Tue Feb 25 15:39:28 2020 +0000 + + Use ByteBuffersDirectory rather than RAMDirectory (#52768) + + Lucene's RAMDirectory has been deprecated. This commit replaces all uses of + RAMDirectory in elasticsearch with the newer ByteBuffersDirectory. Most uses + are in tests, but the percolator and painless executor may get some small speedups. + +commit 02b23c37d12b4819221529e3244f3505b73b96c8 +Author: Nik Everett +Date: Tue Feb 25 10:02:59 2020 -0500 + + Another test fix + + Another attempt to fix a test that fails rarely and randomly. This time + try locking the query to just a single index. + +commit a6f5b4bb786be32bfadcf3f7065f0bd77f9861e3 +Author: Aleksandr Maus +Date: Tue Feb 25 10:22:07 2020 -0500 + + Unmute EqlActionIT (#52757) + + Related to https://github.com/elastic/elasticsearch/issues/52737 + +commit 563f03351135e2cb003e89aa753ebec9eaa00945 +Author: Pius +Date: Tue Feb 25 07:17:07 2020 -0800 + + Update ilm-settings.asciidoc (#51577) + +commit 3ad1783a4170e56cd69e2273b279e70ec3917580 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Feb 25 15:53:46 2020 +0100 + + Delete by query test on low free disk block (#52759) + + The block setup by the test could be released by the nodes cluster info + thread before the disk threshold decider was disabled, now disable + decider first. + +commit d2db16e046bbcb4b4fc177a85dd0e59c2ac3f311 +Author: bellengao +Date: Tue Feb 25 22:35:26 2020 +0800 + + [DOCS] Correct policy name in ILM docs example (#52354) + + Updates an example snippet to use a consistent policy name. + +commit 6c6ab8fa47ce664b82cf109a3a54f9753a45be07 +Author: David Pilato +Date: Tue Feb 25 14:47:58 2020 +0100 + + [DOS] Fix typo in CSV processor docs (#52649) + + Corrects an example array in a snippet of the CSV processor docs. + +commit 49f37989c44803ee6a1d9b974436e0e66523f922 +Author: bellengao +Date: Tue Feb 25 20:51:02 2020 +0800 + + [DOCS] Fix typo in ingest node docs (#52671) + +commit cf122d13b8eb908fe3137f80cd7278ef76bebd8a +Author: David Roberts +Date: Tue Feb 25 12:18:53 2020 +0000 + + [ML] Use event.timezone in file_structure_finder ingest pipeline (#52720) + + This is because beat.timezone was renamed to event.timezone in + elastic/beats#9458 + +commit 9b05f6a66889c22a125fc5d37ff2918849d7a4f6 +Author: James Rodewig +Date: Tue Feb 25 07:19:35 2020 -0500 + + [DOCS] Add admonition for app using cat APIs (#52727) + + Adds an explicit "important" admonition discouraging apps from using + cat APIs. + + cat APIs are intended for human consumption via the command line or + Kibana console only. They are not intended for consumption by + applications. + +commit b2cb38ccf562586e138b710c35d2e86b6906d008 +Author: Aleksandr Maus +Date: Tue Feb 25 07:19:33 2020 -0500 + + EQL: Expand verification tests (#52664) (#52725) + + * EQL: Expand verification tests (#52664) + + Expand verification tests + Fix some error messaging consistency in EqlParser + + Related to https://github.com/elastic/elasticsearch/issues/51873 + + * Adjust for 7.x compatibility + +commit 18663b0a85d54a7908ebd105c1d68088168c21db +Author: Alan Woodward +Date: Tue Feb 25 12:00:23 2020 +0000 + + Don't index ranges including NOW in percolator (#52748) + + Currently, date ranges queries using NOW-based date math are rewritten to + MatchAllDocs queries when being preprocessed for the percolator. However, + since we added the verification step, this can result in incorrect matches when + percolator queries are run without scores. This commit changes things to instead + wrap date queries that use NOW with a new DateRangeIncludingNowQuery. + This is a simple wrapper query that returns its delegate at rewrite time, but it can + be detected by the percolator QueryAnalyzer and be dealt with accordingly. + + This also allows us to remove a method on QueryRewriteContext, and push all + logic relating to NOW-based ranges into the DateFieldMapper. + + Fixes #52617 + +commit 1a14ae4e1b530491d6207c50bdacec85d53a9994 +Author: James Rodewig +Date: Tue Feb 25 07:12:34 2020 -0500 + + [DOCS] Document `include_in_*` nested mapping parms (#52648) + + Adds documentation for the `include_in_parent` and `include_in_root` + mapping parameters for the `nested` mapping datatype. + +commit e2aa5bc174d8e4418bfe6072288ce7fde0e99c1e +Author: Ioannis Kakavas +Date: Tue Feb 25 11:59:13 2020 +0200 + + Adjust file realm docs (#52471) (#52745) + + The existing wording in the file realm docs proved confusing + for users as it seemed to indicate that it should _only_ be + used as a fallback/recovery realm and that it is not a + first class realm. + + This change attempts to clarify this and point out that recovery + is _a_ use case for the file realm but not the only intended one. + +commit 5f81906fcf14d74f9453fb8b519af81ec1a6bd37 +Author: Adrien Grand +Date: Tue Feb 25 08:52:53 2020 +0100 + + Discourage from opting in for the `niofs` store. (#52638) + + Indices open with the `niofs` store type load much more data on-heap than + indices open with the `mmapfs` store type. This limitation is now documented + and examples have been updated to show how to update settings to use the + `mmapfs` store type rather than `niofs`. + +commit 9b0ddc1c033216a03432ad636582237cd0094608 +Author: Adrien Grand +Date: Tue Feb 25 08:52:33 2020 +0100 + + Clarify the resiliency trade-off of disabling replicas to speed up indexing. (#52714) + + We should be more explicit about the downsides of disabling replicas and + explain that users should be ready to re-do the entire load in case of + issues mid-way. + +commit 5ce66b8b3c9d12dc0eb7b5eecbbb09f401cde287 +Author: Adrien Grand +Date: Tue Feb 25 08:52:05 2020 +0100 + + Document how CCR may be used to speed up indexing. (#52717) + + One architecture that we have recommended to several users to speed up + indexing involved using CCR to prevent searching from stealing resources + from indexing. + +commit 28d4b719474ce8dbc9769a99db55c519ca5e490c +Author: Bob Blank <39975390+BobBlank12@users.noreply.github.com> +Date: Mon Feb 24 20:01:14 2020 -0600 + + Clarified http.max_content_length description (#52329) + + Adding "greater than" based on discussion with @jasontedor for clarity. + +commit ac654692d4af96fd0a7d14077642665c8d4a1788 +Author: Mark Vieira +Date: Mon Feb 24 16:34:32 2020 -0800 + + Upgrade to Gradle 6.2.1 (#52732) + +commit 025352f0a41a7f39249a12840db06c686f566b78 +Author: Mark Vieira +Date: Mon Feb 24 16:06:30 2020 -0800 + + Mute EqlActionIT + +commit ed6b10bc038d7658dca157bcf4d74625855ca9b2 +Author: Andrei Stefan +Date: Tue Feb 25 01:41:51 2020 +0200 + + SQL: use a calendar interval for histograms over 1 month intervals (#52586) (#52715) + + (cherry picked from commit 928b11a34ec92d90d082abdf4fa09f7ce1d7c0c4) + +commit ba0401ecfdd83ae89cc04cfb5af19525cc5191ab +Author: Julie Tibshirani +Date: Mon Feb 24 14:57:49 2020 -0800 + + Correct the name of the search timeout parameter. (#52733) + + The request body parameter is called 'timeout', not 'search_timeout'. + +commit d48870ef941910292e5e133476b598aa46d35e82 +Author: Nik Everett +Date: Mon Feb 24 16:12:53 2020 -0500 + + Try to fix test another way..... + + Explictly create the index rather than skip adding the default + template.... + +commit a1e7429ccc0f00fe02370314d2f2c029e9211727 +Author: Ryan Ernst +Date: Mon Feb 24 13:33:50 2020 -0800 + + Allow sha512 checksum without filename for maven plugins (#52668) + + When installing plugins from remote sources, either the Elastic download + service, or maven, a checksum file is downloaded and checked against the + downloaded zip. The current format for official plugins is to use a + sha512 checksum which includes the zip filename. This format matches + that from sha512sum, and allows using the --check argument there to + verify the checksum manually. However, when generating checksum files + with maven and gradle, the filename is not included. + + This commit relaxes the requirement the filename existing within the + sha512 checksum file for maven plugins. We continue to strictly enforce + official plugins have the existing format of the file. + + closes #52413 + +commit 4cae4ded4b8c723e3fb627e281fa4db86dfaf25e +Author: David Roberts +Date: Mon Feb 24 21:35:32 2020 +0000 + + [TEST] Unmute DebMetadataTests.test05CheckLintian (#52719) + + The underlying problem was fixed in elastic/ml-cpp#1019 + + Backport of #52696 + +commit c6e35b460ecde52ca4092243e3fc9b786154fc05 +Author: lcawl +Date: Mon Feb 24 10:03:45 2020 -0800 + + [DOCS] Adds anchor for custom rules + +commit a7fe3329cb87fae215afb6c7e9a2aaf44462f2fb +Author: Nik Everett +Date: Mon Feb 24 14:30:37 2020 -0500 + + Fix some top_metrics tests (#52575) (#52726) + + These tests didn't work properly when run against multi-shard indices. + The `_score` based sorting test expects fairly specific scores which + isn't going to happen with multiple shards so this disables multiple + shards for that test. The other tests were failing due to a fairly + sneaky race condition around `_bulk` and type inference. This fixes them + by always sending metric values as floating point numbers so + Elasticsearch always infers them to be doubles. + +commit 5fba8cbc7b5010a4deec9556425d20655908ed3c +Author: Ryan Ernst +Date: Mon Feb 24 11:14:29 2020 -0800 + + Rename local Environment var in Node to avoid confusion (#52602) + + When the Node class is being constructed, an initial environment is + passed in with the initial settings for the node. Once the plugin + servicie is initialized, the final Environment+Settings are created, at + which point the initial environment should no longer be used. This + commit renames the constructor arg to avoid naming clashes with the + final environment variable. + +commit 8c295cdc8789b199eaa6d8d8f293c1521b463463 +Author: Ryan Ernst +Date: Mon Feb 24 11:13:06 2020 -0800 + + Fix sql cli sourcing of x-pack-env (#52613) + + The sql-cli script sources x-pack-env, but it does so assuming the + current directory is ES_HOME. This commit alters the source command to + use ES_HOME which is available after running elasticsearch-env. + + closes #47803 + +commit 7d9de8412a8eb07b2d1c552c3fa4bbcdc94e1a3a +Author: Lee Hinman +Date: Mon Feb 24 11:57:01 2020 -0700 + + [7.x] fix npe in RestPluginsAction (#52620) (de56de9a) (#52721) + + Relates #45321 + + Co-authored-by: Elastic Machine + + Co-authored-by: Kaihong.Wang + Co-authored-by: Elastic Machine + +commit 034b1c0ba3f40adac30986d86b6ab24abe607bc8 +Author: Mayya Sharipova +Date: Mon Feb 24 13:48:21 2020 -0500 + + Correct boost calculation in script_score query (#52478) (#52724) + + Before boost in script_score query was wrongly applied only to the subquery. + This commit makes sure that the boost is applied to the whole score + that comes out of script. + + Closes #48465 + +commit a7bdb0b4565ef34fefdf6d7c277f00bf95ff2881 +Author: Aleksandr Maus +Date: Mon Feb 24 12:46:59 2020 -0500 + + EQL: Add integration tests harness to test EQL feature parity with original implementation (#52248) (#52675) + + The tests use the original test queries from + https://github.com/endgameinc/eql/blob/master/eql/etc/test_queries.toml + for EQL implementation correctness validation. + The file test_queries_unsupported.toml serves as a "blacklist" for the + queries that we do not support. Currently all of the queries are + blacklisted. Over the time the expectation is to eventually have an + empty "blacklist" when all of the queries are fully supported. + + The tests use the original test vector from + https://raw.githubusercontent.com/endgameinc/eql/master/eql/etc/test_data.json. + + Only one EQL and the response is stubbed for now to match the expected + output from that query. This part would need some tweaking after EQL is + fully wired. + + Related to https://github.com/elastic/elasticsearch/issues/49581 + +commit e72cb7947654969cf6d141c1656f082dbac86fe2 +Author: Przemko Robakowski +Date: Mon Feb 24 18:22:09 2020 +0100 + + Add docs for errors in GetAlias API (#51850) (#52716) + + Closes #31499 + + Co-authored-by: Maxim + +commit f993ef80f883e7746d2044b47ae13bc4e50a004a +Author: Adrien Grand +Date: Mon Feb 24 18:14:12 2020 +0100 + + Move the terms index of `_id` off-heap. (#52518) + + In #42838 we moved the terms index of all fields off-heap except the + `_id` field because we were worried it might make indexing slower. In + general, the indexing rate is only affected if explicit IDs are used, as + otherwise Elasticsearch almost never performs lookups in the terms + dictionary for the purpose of indexing. So it's quite wasteful to + require the terms index of `_id` to be loaded on-heap for users who have + append-only workloads. Furthermore I've been conducting benchmarks when + indexing with explicit ids on the http_logs dataset that suggest that + the slowdown is low enough that it's probably not worth forcing the terms + index to be kept on-heap. Here are some numbers for the median indexing + rate in docs/s: + + | Run | Master | Patch | + | --- | ------- | ------- | + | 1 | 45851.2 | 46401.4 | + | 2 | 45192.6 | 44561.0 | + | 3 | 45635.2 | 44137.0 | + | 4 | 46435.0 | 44692.8 | + | 5 | 45829.0 | 44949.0 | + + And now heap usage in MB for segments: + + | Run | Master | Patch | + | --- | ------- | -------- | + | 1 | 41.1720 | 0.352083 | + | 2 | 45.1545 | 0.382534 | + | 3 | 41.7746 | 0.381285 | + | 4 | 45.3673 | 0.412737 | + | 5 | 45.4616 | 0.375063 | + + Indexing rate decreased by 1.8% on average, while memory usage decreased + by more than 100x. + + The `http_logs` dataset contains small documents and has a simple + indexing chain. More complex indexing chains, e.g. with more fields, + ingest pipelines, etc. would see an even lower decrease of indexing rate. + +commit de3d674bb7949fe44ee1a2592a48d4317fbd1481 +Author: David Kyle +Date: Mon Feb 24 15:22:49 2020 +0000 + + Revert "Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart" + + This reverts commit c4d91143acc8edaf2895b1d464510e92eb7e16a2. + +commit 044a4e127a20d7c1337685cc0d76e2c0634c77f5 +Author: David Kyle +Date: Mon Feb 24 15:21:51 2020 +0000 + + [ML] Add reason to DataFrameAnalyticsTask setFailed log message (#52659) (#52707) + +commit 5e488115851c6d78fde9c9e90b693dfcd9bb8557 +Author: James Rodewig +Date: Mon Feb 24 09:54:33 2020 -0500 + + [DOCS] Document CCS-supported APIs (#52708) + + Explicitly notes the Elasticsearch API endpoints that support CCS. + + This should deter users from attempting to use CCS with other API + endpoints, such as `GET /_doc/<_id>`. + +commit 7dc41a3b831a2bdf1b3303a05e306a84d42cae56 +Author: Alan Woodward +Date: Mon Feb 24 14:28:51 2020 +0000 + + Use BoostQuery rather than FunctionScoreQuery for query-time indices_boost (#52272) + + This is a trivial change, but it should result in a slightly more efficient query boost. + +commit 33131e2dcd0fa984a2e438005c49c45655805032 +Author: Albert Zaharovits +Date: Mon Feb 24 16:38:16 2020 +0200 + + Logfile audit settings validation (#52537) + + Add validation for the following logfile audit settings: + + xpack.security.audit.logfile.events.include + xpack.security.audit.logfile.events.exclude + xpack.security.audit.logfile.events.ignore_filters.*.users + xpack.security.audit.logfile.events.ignore_filters.*.realms + xpack.security.audit.logfile.events.ignore_filters.*.roles + xpack.security.audit.logfile.events.ignore_filters.*.indices + + Closes #52357 + Relates #47711 #47038 + Follows the example from #47246 + +commit ba9d3c63899ecd6a15063d38468d258c6ff5c4d9 +Author: Ignacio Vera +Date: Mon Feb 24 13:46:51 2020 +0100 + + Add support for multipoint shape queries (#52564) (#52705) + +commit 98bcf06bae1cbe7b29c2dcc7ba07097d0f3fafb8 +Author: James Rodewig +Date: Mon Feb 24 07:41:53 2020 -0500 + + [DOCS] Correct multi search API docs (#52523) + + * Adds an example request to the top of the page. + * Relocates several parameters erroneously listed under "Request body" + to the appropriate "Query parameters" section. + * Updates the "Request body" section to better document the NDJSON + structure of msearch requests. + +commit c03f51f68ff3254bd464c63b5e0083b5f76feb85 +Author: Marios Trivyzas +Date: Mon Feb 24 11:57:32 2020 +0100 + + [Docs] Clarify default value for `allow_no_indices` (#52635) (#52697) + + Add default value to each one of the usages of `allow_no_indices` + since it differs between different APIs. + + Relates to: #52534 + + (cherry picked from commit 2eb986488ac326d6da6ab8ad0203a94e08684a36) + +commit 225d841212f727f52566ca060bed77690e01a03c +Author: Martijn van Groningen +Date: Mon Feb 24 10:22:48 2020 +0100 + + Improve watcher test by preventing a npe when closing the http client. + +commit 7cefba78c53e811663a5462695e04be1993fb0d5 +Author: Yang Wang +Date: Mon Feb 24 11:02:40 2020 +1100 + + License removal leads back to a basic license (#52407) (#52683) + + A new basic license will be generated when existing license is deleted. + In addition, deleting an existing basic license is a no-op. + + Resolves: #45022 + +commit d26d7721ea2ca14c578c55a769f496917b7006d6 +Author: Nik Everett +Date: Sun Feb 23 17:13:55 2020 -0500 + + Continue realizing sorting by aggregations (backport of #52298) (#52667) + + This drops more of the `instanceof`s from `AggregationPath`. There are + still a couple in `AggregationPath`. And I ended up moving two into + `BucketsAggregator`, but I think this is still an improvement! + +commit a0aa808c83e02e24ad1f128faf2b13213f276a07 +Author: Mark Vieira +Date: Sun Feb 23 08:46:07 2020 -0800 + + Fix broken BWC builds + +commit 72a2d0f9d89851827517e56806c7d7d9f09a475d +Author: Mark Vieira +Date: Sat Feb 22 18:17:15 2020 -0800 + + Skip 'setupPorts' tasks when Docker is unavailable (#52679) + +commit 02cb5b6c0e106fd6a0ccb5b7298f42f500e7f3c1 +Author: bellengao +Date: Sat Feb 22 20:35:05 2020 +0800 + + Return 429 status code on read_only_allow_delete index block (#50166) + + We consider index level read_only_allow_delete blocks temporary since + the DiskThresholdMonitor can automatically release those when an index + is no longer allocated on nodes above high threshold. + + The rest status has therefore been changed to 429 when encountering this + index block to signal retryability to clients. + + Related to #49393 + +commit 1685cbe504f58c07409560b7b9476e07964d2801 +Author: Jason Tedor +Date: Sat Feb 22 09:09:04 2020 -0500 + + Add messages for CCR on license state changes (#52470) + + When a license expires, or license state changes, functionality might be + disabled. This commit adds messages for CCR to inform users that CCR + functionality will be disabled when a license expires, or when license + state changes to a license level lower than trial/platinum/enterprise. + +commit afd90647c9d2633909691f3b000dd42b2ed3303c +Author: Benjamin Trent +Date: Fri Feb 21 18:42:31 2020 -0500 + + [ML] Adds feature importance to option to inference processor (#52218) (#52666) + + This adds machine learning model feature importance calculations to the inference processor. + + The new flag in the configuration matches the analytics parameter name: `num_top_feature_importance_values` + Example: + ``` + "inference": { + "field_mappings": {}, + "model_id": "my_model", + "inference_config": { + "regression": { + "num_top_feature_importance_values": 3 + } + } + } + ``` + + This will write to the document as follows: + ``` + "inference" : { + "feature_importance" : { + "FlightTimeMin" : -76.90955548511226, + "FlightDelayType" : 114.13514762158526, + "DistanceMiles" : 13.731580450792187 + }, + "predicted_value" : 108.33165831875137, + "model_id" : "my_model" + } + ``` + + This is done through calculating the [SHAP values](https://arxiv.org/abs/1802.03888). + + It requires that models have populated `number_samples` for each tree node. This is not available to models that were created before 7.7. + + Additionally, if the inference config is requesting feature_importance, and not all nodes have been upgraded yet, it will not allow the pipeline to be created. This is to safe-guard in a mixed-version environment where only some ingest nodes have been upgraded. + + NOTE: the algorithm is a Java port of the one laid out in ml-cpp: https://github.com/elastic/ml-cpp/blob/master/lib/maths/CTreeShapFeatureImportance.cc + + usability blocked by: https://github.com/elastic/ml-cpp/pull/991 + +commit f06d692706ee4ffbb851a2db3c8ba1f1a4c129c5 +Author: Mark Vieira +Date: Fri Feb 21 15:24:05 2020 -0800 + + [Backport] Consolidate docker availability logic (#52656) + +commit 8abfda0b598b4be67c50b20706ff204b5221b352 +Author: Jay Modi +Date: Fri Feb 21 13:30:11 2020 -0700 + + Rename assertThrows to prevent naming clash (#52651) + + This commit renames ElasticsearchAssertions#assertThrows to + assertRequestBuilderThrows and assertFutureThrows to avoid a + naming clash with JUnit 4.13+ and static imports of these methods. + Additionally, these methods have been updated to make use of + expectThrows internally to avoid duplicating the logic there. + + Relates #51787 + Backport of #52582 + +commit 3840a763d8a8e060a729c052ce2db4148970fc8c +Author: Mayya Sharipova +Date: Fri Feb 21 14:56:16 2020 -0500 + + Correct release notes for 7.5 (#52660) + + Remove a mention to a feature that was not merged, + as its corresponding PR was closed. + +commit ce7ebb2d3905220ef3a9e72202fa8d5c7df1a4f5 +Author: Rory Hunter +Date: Fri Feb 21 19:36:15 2020 +0000 + + Limit _FILE env var support to specific vars (#52645) + + Backport of #52525. + + Closes #52503. Implement a list of `_FILE` env vars that will be used to + populate env vars with file content, instead of processing all `_FILE` + vars in the environment. + +commit 376932a47dc43442ba2d40ddc7ea4054ef2c74c2 +Author: Stuart Tettemer +Date: Fri Feb 21 12:10:51 2020 -0700 + + Scripting: split out compile limits and caching (#52498) (#52652) + + Phase 1 of adding compilation limits per context. + * Refactor rate limiting and caching into separate class, + `ScriptCache`, which will be used per context. + * Disable compilation limit for certain tests. + + Backport of 0866031 + Refs: #50152 + +commit 56efd8b44d19fadb6475bf9912e33779f1e4501c +Author: Lisa Cawley +Date: Fri Feb 21 10:04:29 2020 -0800 + + [DOCS] Adds certutil http command to TLS setup steps (#51241) + + Co-Authored-By: Ioannis Kakavas + Co-Authored-By: Tim Vernum + +commit c4d91143acc8edaf2895b1d464510e92eb7e16a2 +Author: Jack Conradson +Date: Fri Feb 21 09:24:23 2020 -0800 + + Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart + + Relates: #52654 + +commit 101bca86d2c32b5674f6244e99e5f591b2a3e25b +Author: Nik Richers +Date: Fri Feb 21 09:04:39 2020 -0800 + + [DOCS] Switch to standard ESS trial links (#52552) + + Switches ESS trial sign-up links over to a standard attribute. This provides better metrics for how effective these links are. + +commit 4ff78e8a00ca75292921f987ecd32b647e585d7a +Author: Lisa Cawley +Date: Fri Feb 21 06:57:11 2020 -0800 + + [7.x][DOCS] Adds X-Pack usage API (#52592) + +commit f3f6ff97eeb37076f92732524d5522697726992e +Author: Jay Modi +Date: Fri Feb 21 07:50:02 2020 -0700 + + Single instance of the IndexNameExpressionResolver (#52604) + + This commit modifies the codebase so that our production code uses a + single instance of the IndexNameExpressionResolver class. This change + is being made in preparation for allowing name expression resolution + to be augmented by a plugin. + + In order to remove some instances of IndexNameExpressionResolver, the + single instance is added as a parameter of Plugin#createComponents and + PersistentTaskPlugin#getPersistentTasksExecutor. + + Backport of #52596 + +commit ed957f35a94703505c25315b6f8a16d42f16e820 +Author: Nik Everett +Date: Fri Feb 21 09:49:17 2020 -0500 + + Cover missing case in top_metrics test (#52517) + + The top_metrics test assumed that it'd never end up *only* reducing + unmapped results. But, rarely, it does. This handles that case in the + test. + + Closes #52462 + +commit e5b21a3fc6e0cdeb9343aecacf8b07e5a5ff9192 +Author: Igor Motov +Date: Fri Feb 21 22:44:08 2020 +0900 + + Add HLRC for EQL search (#52550) + + Adds EQL HLRC client with the search method. + + Relates to #51961 + +commit 068181b0b68461ef1ca6f5cf67d95e1e5621ce60 +Author: James Rodewig +Date: Fri Feb 21 08:07:28 2020 -0500 + + [DOCS] Add missing `indices` parms returned by `_nodes/stats` (#52055) + + Adds several human-readable `indices` parameters returned by the + `_nodes/stats` API. + +commit 288ccae23b5941e17a379f8da70deade1e47e719 +Author: Hendrik Muhs +Date: Fri Feb 21 13:06:18 2020 +0100 + + [Transform] add support for filter aggregation (#52483) + + add support for filter aggregations, refactor code for sub-aggregation support in mapping + deduction + + fixes #52151 + +commit 7fe2843a9ef7a6c383ba5e30fbf9ebb71dacdfc3 +Author: Andrei Stefan +Date: Fri Feb 21 13:26:31 2020 +0200 + + SQL: specify command to run the CLI on a remote machine without Elasticsearch (#52626) + + (cherry picked from commit 477b0eda8322c5dcb6861bd262bfeec17ff133fe) + +commit 80b77e92d4faf8f96001d715473e8585b775e958 +Author: James Rodewig +Date: Fri Feb 21 05:32:10 2020 -0500 + + [7.x] [DOCS] Re-add redirects for API relocation (#52628) + + Re-adds several redirects removed with #50510. + + These redirects were related to the relocation of several API docs to + new pages under the 'REST APIs' chapter. + + We've since decided to only remove such redirects with major releases. + +commit 96d603979b4a50b795c3c12a5bf5e69bc96d4d12 +Author: markharwood +Date: Fri Feb 21 10:25:03 2020 +0000 + + Upgrade Lucene to 8.5.0-snapshot-b01d7cb (#52584) + + Upgrading 7x to same Lucene 8.5 version used in master + +commit 5a7db0c5203153f85f9a3678577e64ec788e2567 +Author: Armin Braun +Date: Fri Feb 21 10:34:53 2020 +0100 + + Fix GCS Test testReadLargeBlobWithRetries (#52619) (#52624) + + The countdown didn't work well here because it only returns `true` once the countdown reaches `0` + but can on subsequent executions return `false` again if a countdown at `0` is counted down again, + leading to more than the expected number of simulated failures. + + Closes #52607 + +commit 5017bb094eb07cbc360e9dae332b92431b38f460 +Author: Sean Story +Date: Fri Feb 21 03:23:56 2020 -0600 + + [Docs]: Fix typo 'Got' -> 'Go' (#52603) + + Fix typo 'Got' -> 'Go' + + (cherry picked from commit cf7eca270db964c9c474a70da647cb8396f677ba) + +commit 1662cd45a4923d1e9db131d2a29a8f1b22ce85fe +Author: Armin Braun +Date: Fri Feb 21 10:21:20 2020 +0100 + + Add Region and Signer Algorithm Overrides to S3 Repos (#52112) (#52562) + + Exposes S3 SDK signing region and algorithm override settings as requested in #51861. + + Closes #51861 + +commit 0a09e15959439a81b5d2e571998778807de61c0f +Author: Armin Braun +Date: Fri Feb 21 10:20:07 2020 +0100 + + Add Caching for RepositoryData in BlobStoreRepository (#52341) (#52566) + + Cache latest `RepositoryData` on heap when it's absolutely safe to do so (i.e. when the repository is in strictly consistent mode). + + `RepositoryData` can safely be assumed to not grow to a size that would cause trouble because we often have at least two copies of it loaded at the same time when doing repository operations. Also, concurrent snapshot API status requests currently load it independently of each other and so on, making it safe to cache on heap and assume as "small" IMO. + + The benefits of this move are: + * Much faster repository status API calls + * listing all snapshot names becomes instant + * Other operations are sped up massively too because they mostly operate in two steps: load repository data then load multiple other blobs to get the additional data + * Additional cloud cost savings + * Better resiliency, saving another spot where an IO issue could break the snapshot + * We can simplify a number of spots in the current code that currently pass around the repository data in tricky ways to avoid loading it multiple times in follow ups. + +commit aff693bc9f12fbee1e1ab307c8c3383d5d3cfbcd +Author: Przemko Robakowski +Date: Fri Feb 21 10:11:35 2020 +0100 + + Make FreezeStep retryable (#52540) (#52559) + + * Make FreezeStep retryable + + This change marks `FreezeStep` as retryable and adds test to make sure we can really run it again. + + * refactor tests + + Co-authored-by: Elastic Machine + +commit 4bb780bc3736924df97d2f11367988ee777e6358 +Author: Armin Braun +Date: Fri Feb 21 09:14:34 2020 +0100 + + Refactor Inflexible Snapshot Repository BwC (#52365) (#52557) + + * Refactor Inflexible Snapshot Repository BwC (#52365) + + Transport the version to use for a snapshot instead of whether to use shard generations in the snapshots in progress entry. This allows making upcoming repository metadata changes in a flexible manner in an analogous way to how we handle serialization BwC elsewhere. + Also, exposing the version at the repository API level will make it easier to do BwC relevant changes in derived repositories like source only or encrypted. + +commit b84e8db7b5a3da90444afec830e210920e0c164f +Author: Przemysław Witek +Date: Fri Feb 21 08:55:59 2020 +0100 + + [7.x] Rename .ml-state index to .ml-state-000001 to support rollover (#52510) (#52595) + +commit 1d895118ddf2d3d2bf08f47f1ee890af00340e58 +Author: István Zoltán Szabó +Date: Fri Feb 21 08:22:04 2020 +0100 + + [DOCS] Links transforms in aggregation docs (#52563) + + Co-authored-by: Lisa Cawley + +commit c9b7bb282a00e669d88a98dd461bdee3b5f5c3aa +Author: Andrei Stefan +Date: Fri Feb 21 09:21:44 2020 +0200 + + Move IsNull/IsNotNull predicates to QL project (#52502) (#52546) + + (cherry picked from commit b7d534e20c005f1c3565e52c0d0e0273f4a4cece) + +commit 107f00a4ec47e4aef9f1c55c227c287391e1bee2 +Author: Ignacio Vera +Date: Fri Feb 21 07:45:53 2020 +0100 + + Add support for multipoint geoshape queries (#52133) (#52553) + + Currently multi-point queries are not supported when indexing your data using BKD-backed geoshape strategy. This commit removes this limitation. + +commit 4bc7545e4333ce0ee4500f1b8367b9d79a48fb05 +Author: Yang Wang +Date: Fri Feb 21 14:18:18 2020 +1100 + + Add enterprise mode and refactor license check (#51864) (#52115) + + Add enterprise operation mode to properly map enterprise license. + + Aslo refactor XPackLicenstate class to consolidate license status and mode checks. + This class has many sychronised methods to check basically three things: + * Minimum operation mode required + * Whether security is enabled + * Whether current license needs to be active + + Depends on the actual feature, either 1, 2 or all of above checks are performed. + These are now consolidated in to 3 helper methods (2 of them are new). + The synchronization is pushed down to the helper methods so actual checking + methods no longer need to worry about it. + + resolves: #51081 + +commit 2a5c181dda1d62dad9131a1b88cab0c494c16973 +Author: Benjamin Trent +Date: Thu Feb 20 19:47:29 2020 -0500 + + [ML][Inference] don't return inflated definition when storing trained models (#52573) (#52580) + + When `PUT` is called to store a trained model, it is useful to return the newly create model config. But, it is NOT useful to return the inflated definition. + + These definitions can be large and returning the inflated definition causes undo work on the server and client side. + + Co-authored-by: Elastic Machine + +commit 4bce9984e6119d7126a77f678435c094c47094fd +Author: Mark Vieira +Date: Thu Feb 20 15:13:11 2020 -0800 + + Mute GoogleCloudStorageBlobContainerRetriesTests.testReadLargeBlobWithRetries + + Signed-off-by: Mark Vieira + +commit 013d5c2d24b6dbfcf4ef9a5c4fbeea80ff48f43c +Author: Benjamin Trent +Date: Thu Feb 20 17:22:59 2020 -0500 + + [ML] Adds support for a global calendar via `_all` (#50372) (#52578) + + This adds `_all` to Calendar searches. This enables users to supply the `_all` string in the `job_ids` array when creating a Calendar. That calendar will now be applied to all jobs (existing and newly created). + + Closes #45013 + + Co-authored-by: Elastic Machine + +commit a8725cde53c75814ec3d1e1a0559925c682258b7 +Author: Mark Vieira +Date: Thu Feb 20 12:48:45 2020 -0800 + + Mute DebMetadataTests.test05CheckLintian + + Signed-off-by: Mark Vieira + +commit 8fb9bed07875badffedf56e599a14e9410ccaf9a +Author: Rory Hunter +Date: Thu Feb 20 14:39:44 2020 +0000 + + Fix compilation error + +commit ba8d6d1fb52e7460cc914bdf4d3aa4dde4980094 +Author: Maria Ralli +Date: Thu Feb 20 16:06:45 2020 +0200 + + Remove Xlint exclusions from gradle files + + Backport of #52542. + + This commit is part of issue #40366 to remove disabled Xlint warnings + from gradle files. In particular, it removes the Xlint exclusions from + the following files: + + - benchmarks/build.gradle + - client/client-benchmark-noop-api-plugin/build.gradle + - x-pack/qa/rolling-upgrade/build.gradle + - x-pack/qa/third-party/active-directory/build.gradle + - modules/transport-netty4/build.gradle + + For the first three files no code adjustments were needed. For + x-pack/qa/third-party/active-directory move the suppression at the code + level. For transport-netty4 replace the variable arguments with + ArrayLists and remove any redundant casts. + +commit d76358c875d0686974e847b172671664d0074e48 +Author: Yannick Welsch +Date: Thu Feb 20 11:11:06 2020 +0100 + + Deprecate fixed_auto_queue_size thread pool type (#52399) + + Relates #52280 + +commit 087ceb899b653a055b9278187e01ab3ff7514637 +Author: Russ Cam +Date: Thu Feb 20 14:39:15 2020 +1100 + + Reinstate params in rest api specs (#52544) + + This commit reinstates the following params in the rest specs: + + 1. "analyzer" in delete_by_query + 2. "ccs_minimize_roundtrips" in msearch_template + 3. "ccs_minimize_roundtrips" in search_template + + All appear to be valid options that seem to have been inadvertantly removed + between 7.3 and 7.4. + + Fixes elastic/elasticsearch#47768 + +commit 62da077beb1ce8fec8f66411a2418970e052fbcb +Author: Russ Cam +Date: Thu Feb 20 12:33:06 2020 +1100 + + Specify name on enrich.get_policy as list type (#50217) + + This commit updates the enrich.get_policy API to specify name + as a list, in line with other URL parts that accept a comma-separated + list of values. + + In addition, update the get enrich policy API docs + to align the URL part name in the documentation with + the name used in the REST API specs. + + (cherry picked from commit 94f6f946ef283dc93040e052b4676c5bc37f4bde) + +commit 3c3a0b2f37d9b3917f56dc01694f578020bed827 +Author: Ryan Ernst +Date: Wed Feb 19 16:08:36 2020 -0800 + + Mute additional failing top_metrics test (#52545) + + Most top_metrics tests were muted in #52468, but the scaled float can + also fail. This commit mutes that test as well. + + relates #52418 + +commit 8b1ad98219e5273da121d9e12692bb68c3c75c47 +Author: Mark Vieira +Date: Wed Feb 19 13:41:50 2020 -0800 + + Disable duplicate project detection in Gradle 6.2 (#52529) + +commit b11dbb22054bb648f861677c61f3e7eb2b34173f +Author: Lee Hinman +Date: Wed Feb 19 13:45:32 2020 -0700 + + Correct SLM retention timezone documentation (#52533) + + This erroneously said that retention is run in the master node's timezone, however, it is actually + run in UTC. + +commit 88bb06f055d9b271f128d0c683934616d21a2412 +Author: Przemko Robakowski +Date: Wed Feb 19 21:16:59 2020 +0100 + + Make DeleteStep retryable (#52494) (#52532) + + * Make DeleteStep retryable + + This change marks `DeleteStep` as retryable and adds test to make sure we really can invoke it again. + + * Fix unused import + + * revert unneeded changes + + * test reworked + +commit 22cf1140eba9552718c11ea3408d5f4b6b56dc27 +Author: Lee Hinman +Date: Wed Feb 19 13:15:01 2020 -0700 + + [7.x] Add additional logging to SLM retention task (#52343) (#52535) + + This commit adds more logging to the actions that the SLM retention task does. It will help in the + event that we need to diagnose any additional issues or problems while running retention. + +commit a68fafd64b78b3a441e77a008d4de77c1169a0d1 +Author: Valentin Crettaz +Date: Wed Feb 19 21:26:47 2020 +0400 + + [DOCS] Clarify that "now" cannot be used in `date_range` at index time (#52446) + + `date_range` fields do not accept `"now"` as a value of either bounds at indexing time. + + This corrects an error in the range data type mapping docs. + +commit aeb7b777e62efa7d79dd2d99b7effc4a6214eefa +Author: Armin Braun +Date: Wed Feb 19 18:29:13 2020 +0100 + + Add Blob Download Retries to GCS Repository (#52479) (#52521) + + * Add Blob Download Retries to GCS Repository + + Exactly as #46589 (and kept as close to it as possible code wise so we can dry things up in a follow-up potentially) but for GCS. + + Closes #52319 + +commit 01ec922d1342c328a788a8cca73e09e9eea5204a +Author: Mark Vieira +Date: Wed Feb 19 09:05:39 2020 -0800 + + Suppress Gradle deprecation warnings in the console output (#52332) + +commit db8b306085be34b5d14eab5778d6039b1d2b92bc +Author: Bogdan Pintea +Date: Wed Feb 19 17:33:48 2020 +0100 + + SQL: update ODBC docs, cover Cloud ID, latest params (#52291) + + * Refresh snapshots with latest look + + Add new snapshots with the connection editor to reflect the latest UI. + + * Document the effect of the late added params + + Add details about the Cloud ID setting, as well as those on the Misc + tab. + + + (cherry picked from commit afa67625e847e99a22264f5dd6fa0daa37786c6f) + +commit 3afb5ca1330b96046af3b509ab05e69b64b63fee +Author: Yannick Welsch +Date: Wed Feb 19 16:07:15 2020 +0100 + + Fix synchronization in ByteSizeCachingDirectory (#52512) + + One particular code place was synchronizing on the wrong object. + +commit 7bbe5c846447ef533e745e888aaf77478b367328 +Author: David Kyle +Date: Wed Feb 19 14:41:43 2020 +0000 + + [Ml] Validate tree feature index is within range (#52514) + + This changes the tree validation code to ensure no node in the tree has a + feature index that is beyond the bounds of the feature_names array. + Specifically this handles the situation where the C++ emits a tree containing + a single node and an empty feature_names list. This is valid tree used to + centre the data in the ensemble but the validation code would reject this + as feature_names is empty. This meant a broken workflow as you cannot GET + the model and PUT it back + +commit 43376c6e061710651fcb3d78274798c159db53d9 +Author: James Rodewig +Date: Wed Feb 19 09:14:22 2020 -0500 + + [DOCS] Document how CCS handles cluster-level settings (#49941) + + Updates the cross-cluster search (CCS) documentation to note how + cluster-level settings are applied. + + When `ccs_minimize_roundtrips` is `true`, each cluster applies its own + cluster-level settings to the request. + + When `ccs_minimize_roundtrips` is `false`, cluster-level settings for + the local cluster is used. This includes shard limit settings, such as + `action.search.shard_count.limit`, `pre_filter_shard_size`, and + `max_concurrent_shard_requests`. If these limits are set too low, the + request could be rejected. + +commit 8796cdce4b425a3de5838dbf8a071b262894489f +Author: Nik Everett +Date: Wed Feb 19 09:20:49 2020 -0500 + + Modernize boxplot's parser (backport of #52361) (#52372) + + Uses a newer way to build `ObjectParser` for in `boxplot` that allows us + to drop a mostly ceremonial method. + +commit 7cd997df84e0e46982f491359e9d34d8d6da2c9c +Author: Przemysław Witek +Date: Wed Feb 19 14:02:32 2020 +0100 + + [ML] Make ml internal indices hidden (#52423) (#52509) + +commit 4d006f09d28cf480ca5d2af4710320b12e5e8932 +Author: Hendrik Muhs +Date: Wed Feb 19 12:31:18 2020 +0100 + + [Transform] fix XPackRestIT continuous transform stats test failure + + do not match explicit number but only test existence for duration test (#52504) + + fixes #52429 + +commit 5acee761eb390e45ac8e712c59afe7c61adbbc61 +Author: Przemysław Witek +Date: Wed Feb 19 12:24:59 2020 +0100 + + Implement unit tests for AnomalyDetectorsIndex class (#52417) (#52508) + +commit 8d2261fe47926a0f824f6963be2d8b32d57af890 +Author: Ignacio Vera +Date: Wed Feb 19 12:04:29 2020 +0100 + + Refactor GeoShapeIndexer by extracting polygon / line decomposers (#52422) (#52506) + + Refactor GeoShapeIndexer. We extract Polygon and Line decomposers which are in charge of breaking a shape around the dateline if needed. + +commit 9d40277d4cb92982f6e29768f60216bf541962c2 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Feb 19 11:12:50 2020 +0100 + + Deciders should not by default collect yes'es (#52438) + + AllocationDeciders would collect Yes decisions when not asking for debug + info. Changed to only include Yes decisions when debug is requested + (explain). + +commit d4bc3b75dcb8754a2c1f58d2e525d2c91b47b238 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Feb 19 08:52:08 2020 +0100 + + Reindex: allow comma separated source indices (#52044) + + Added ability to specify comma separated list of source indices without + array. Also fixed so that empty string results in validation error + rather than index does not exist. + + Closes #51949 + +commit baf184c93fbe94597aab0e9de9cba8d13907c717 +Author: David Turner +Date: Wed Feb 19 07:51:04 2020 +0000 + + Avoid using WindowsFS in ClusterRerouteIT (#52488) + + Issue #52000 looks like a case of cluster state updates being slower than + expected, but it seems that these slowdowns are relatively rare: most + invocations of `testDelayWithALargeAmountOfShards` take well under a minute in + CI, but there are occasional failures that take 6+ minutes instead. When it + fails like this, cluster state persistence seems generally slow: most are + slower than expected, with some small updates even taking over 2 seconds to + complete. + + The failures all have in common that they use `WindowsFS` to emulate Windows' + behaviour of refusing to delete files that are still open, by tracking all + files (really, inodes) and validating that deleted files are really closed + first. There is a suggestion that this is a little slow in the Lucene test + framework [1]. To see if we can attribute the slowdown to that common factor, + this commit suppresses the use of `WindowsFS` for this test suite. + + [1] https://github.com/apache/lucene-solr/blob/4a513fa99f638cb65e0cae59bfdf7af410c0327a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java#L166 + +commit 3e3d80e1c81396d00bc3967579be4cfa334a7904 +Author: Mark Vieira +Date: Tue Feb 18 16:24:06 2020 -0800 + + Use actual current commit id for build scan custom value (#52487) + +commit a99d8b71b1cf715d686e1acfdf53c77cdca08cc7 +Author: Mark Vieira +Date: Tue Feb 18 15:34:59 2020 -0800 + + Upgrade to Gradle 6.2 (#51824) + +commit 969cdfaaa4a9e676a77bc3b51dd51657e2240725 +Author: debadair +Date: Tue Feb 18 12:41:05 2020 -0800 + + [DOCS] Clean up links from SQL client app pages. (#52442) + + * [DOCS] Clean up links from SQL client app pages. + + * Linked to client apps from prereqs. + +commit 8038f9bba66a85e2de418d133cba7fba08fbd166 +Author: Tim Brooks +Date: Tue Feb 18 08:09:07 2020 -0700 + + Do not lock when generating time based uuid (#52436) + + Currently we lock when generating time based uuids. The lock is + implemented to prevent concurrent writes to the last timestamp. The uuid + generation is an area of contention when indexing. This commit modifies + the code to use atomic compare and set operations to update the last + timestamp. + +commit 123b3c6f556a00a35fb1cc0a6784851fe9bfbeac +Author: Lisa Cawley +Date: Tue Feb 18 08:48:24 2020 -0800 + + [DOCS] Clarifies description of num_top_feature_importance_values (#52246) + + Co-Authored-By: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> + +commit 7fcd997b3984aa654d9856cf114e2e4b0d3c7fa6 +Author: Tim Brooks +Date: Tue Feb 18 08:08:39 2020 -0700 + + Do not lock on settings keyset if keys initialized (#52435) + + Every time a setting#exist call is made we lock on the keyset to ensure + that it has been initialized. This a heavyweight operation that only + should be done once. This commit moves to a volatile read instead to + prevent unnecessary locking. + +commit b5e191fa573c7bea75fed79ecf040a7c23a2df22 +Author: Tim Brooks +Date: Fri Feb 14 09:49:14 2020 -0700 + + Use thread local random for request id generation (#52344) + + Currently we used the secure random number generate when generating http + request ids in the security AuditUtil. We do not need to be using this + level of randomness for this use case. Additionally, this random number + generator involves locking that blocks the http worker threads at high + concurrency loads. + + This commit modifies this randomness generator to use our reproducible + randomness generator for Elasticsearch. This generator will fall back to + thread local random when used in production. + +commit a742c58d45dfe77848e4603415f3754eecc30dc2 +Author: Tim Brooks +Date: Fri Jan 31 09:43:25 2020 -0700 + + Extract a ConnectionManager interface (#51722) + + Currently we have three different implementations representing a + `ConnectionManager`. There is the basic `ConnectionManager` which + holds all connections for a cluster. And a remote connection manager + which support proxy behavior. And a stubbable connection manager for + tests. The remote and stubbable instances use the delegate pattern, + so this commit extracts an interface for them all to implement. + +commit e752221fc642118de5350d7bc5e8993ea8096d3e +Author: Tim Brooks +Date: Fri Jan 31 10:33:23 2020 -0700 + + Upgrade netty to 4.1.45.Final (#51689) + + Upgrade netty. + +commit 09773efb413c892d90c0c6df59d8bb5a0aa36513 +Author: Ioannis Kakavas +Date: Tue Feb 18 17:16:24 2020 +0200 + + [7.x] Return realm name in SAML Authenticate API (#52188) (#52465) + + This is useful in cases where the caller of the API needs to know + the name of the realm that consumed the SAML Response and + authenticated the user and this is not self evident (i.e. because + there are many saml realms defined in ES). + Currently, the way to learn the realm name would be to make a + subsequent request to the `_authenticate` API. + +commit 0c4f7dc193ee3dd13211f6017f1a290c1343ca87 +Author: Benedict Jin +Date: Tue Feb 18 22:59:26 2020 +0800 + + Minor code improvements (#51921) + + Fix some whitespaces, comments and usage of `this.`. + + (cherry picked from commit 9f59900bf6389172811eb2279c17a2dc7cd9dfdf) + +commit 3d57a78deb2482db8bd3c7aa98b607113ecd0623 +Author: David Turner +Date: Tue Feb 18 13:01:56 2020 +0000 + + Add extra logging for investigation into #52000 (#52472) + + It looks like #52000 is caused by a slowdown in cluster state application + (maybe due to #50907) but I would like to understand the details to ensure that + there's nothing else going on here too before simply increasing the timeout. + This commit enables some relevant `DEBUG` loggers and also captures stack + traces from all threads rather than just the three hottest ones. + +commit 84de601551ec520b23de83f1bbe15000085440da +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Feb 18 13:28:54 2020 +0100 + + Mute failing top_metrics tests (#52468) + + These tests fails when the global template is added, which changes + number_of_shards to 2. + + Relates #52409 and #52418 + +commit 606bc8037f05ae482679c7a7e609ec6e3acca601 +Author: Martijn van Groningen +Date: Tue Feb 18 13:28:15 2020 +0100 + + Adjusted assertion for watcher rolling upgrade test. (#52463) + + Relates to #33185 + +commit d9ce0e673386dab213d6e615c47b29693c4ba762 +Author: Ioannis Kakavas +Date: Tue Feb 18 14:11:34 2020 +0200 + + Update BouncyCastle to 1.64 (#52185) (#52464) + + This commit upgrades the bouncycastle dependency from 1.61 to 1.64. + +commit 9c49868bc5053673fe08b995bf4d1a22a3fbf0da +Author: David Roberts +Date: Tue Feb 18 11:16:54 2020 +0000 + + [TEST] Use busy asserts in ML distributed failure test (#52461) + + When changing a job state using a mechanism that doesn't + wait for the desired state to be reached within the production + code the test code needs to loop until the cluster state has + been updated. + + Closes #52451 + +commit 6fa067a2a08be7a726804b46c29f1bbfe2be0a72 +Author: Przemysław Witek +Date: Tue Feb 18 11:57:03 2020 +0100 + + Relax assertions on memory_estimation.* fields (#52452) (#52458) + +commit 4fb399396ca1d6ba971e81fa17604a4727a45618 +Author: Rory Hunter +Date: Tue Feb 18 10:44:32 2020 +0000 + + Fix mixed threadlocal and supplied random use (#52459) + + Closes #52450. `setRandomIndexSettings(...)` in `ESIntegTestCase` was + using both a thread-local and a supplied source of randomness. Fix this + method to only use the supplied source. + +commit 57d6dd7e3117f3e25eeacf78afbbe7076be67d16 +Author: Armin Braun +Date: Tue Feb 18 11:37:53 2020 +0100 + + Fix Non-Verbose Snapshot List Missing Empty Snapshots (#52433) (#52456) + + We were not including snapshots without indices in the non-verbose + listing because we used the snapshot -> indices mapping to get the + snapshots. + +commit a9c7557ac4440947fafed78e73da2f4e8d6c607a +Author: Armin Braun +Date: Tue Feb 18 11:37:34 2020 +0100 + + Fix Failure to Drain Stream in GCS Repo Tests (#52431) (#52454) + + Same as #51933 but for the custom handler just used in this test. + + Closes #52430 + +commit d467c50e9003bc2dee21efb9e107249ea3a455d1 +Author: Przemko Robakowski +Date: Tue Feb 18 11:01:42 2020 +0100 + + Make TimeSeriesLifecycleActionsIT.testWaitForSnapshot and testWaitForSnapshotSlmExecutedBefore wait for snaphost (#51892) (#52419) + + * waitForSnapshot tests rework + + * Refactor assertBusy + + Co-authored-by: Elastic Machine + + Co-authored-by: Elastic Machine + +commit d17ecb593612010d8465336a2116e85f91969f48 +Author: Martijn van Groningen +Date: Tue Feb 18 10:53:39 2020 +0100 + + Change the delete policy api to not pass wildcard expressions to the delete index api (#52448) + + Backport from #52179 + + Don't rely on the delete index api to resolve all the enrich indices for a particular enrich policy using a '[policy_name]-*' wildcard expression. With this change, the delete policy api will resolve the indices to remove and pass that directly to the delete index api. + + This resolves a bug, that if `action.destructive_requires_name` setting has been set to true then the delete policy api is unable to remove the enrich indices related to the policy being deleted. + + Closes #51228 + + Co-authored-by: bellengao + +commit 2071f85e1aeda07c34e4ac3074edd58edd3f188c +Author: Hendrik Muhs +Date: Tue Feb 18 08:44:37 2020 +0100 + + forward audits to logs (#52394) + + audit messages are stored in the notifications index, so audit information is lost for integration + tests. This change forwards audit messages to logs, so they can help to debug issues. + + relates: #51627 + +commit bdb2e72ea400b24042ed91bf268f23e838caef23 +Author: Nhat Nguyen +Date: Mon Feb 17 15:05:42 2020 -0500 + + Fix timeout in testDowngradeRemoteClusterToBasic (#52322) + + - ESCCRRestTestCase#ensureYellow does not work well with assertBusy + - Increases timeout to 60s + + Closes #52036 + +commit 48ccf36db9930825c3b6c21c0aa5aa4e7ec1695d +Author: David Roberts +Date: Mon Feb 17 16:50:12 2020 +0000 + + [ML] Increase assertBusy timeout in ML node failure tests (#52425) + + Following the change to store cluster state in Lucene indices + (#50907) it can take longer for all the cluster state updates + associated with node failure scenarios to be processed during + internal cluster tests where several nodes all run in the same + JVM. + +commit 20862fe64f57ea39ff82bd4d72a029e48d2e86de +Author: Costin Leau +Date: Mon Feb 17 16:17:14 2020 +0200 + + Break QueryTranslator into QL and SQL (#52397) + + Refactor the code to allow contextual parameterization of dateFormat and + name. + Separate aggs/query implementation though there's room for improvement + in the future + + (cherry picked from commit e086f81b688875b33d01e4504ce7377031c8cf28) + +commit cc628748e10569abc2bc57e65f315145440a8c5a +Author: Armin Braun +Date: Mon Feb 17 13:07:20 2020 +0100 + + Optimize FilterStreamInput for Network Reads (#52395) (#52403) + + When `FilterStreamInput` wraps a Netty `ByteBuf` based stream it + did not forward the bulk primitive reads to the delegate. + These are optimized on the delegate but if they're not forwarded + then the delegate will be called e.g. 4 times to read an `int`. + This happens for essentially all network reads prior to this + change because they all run from a `NamedWritableAwareStreamInput`. + + This also required optimising `BufferedChecksumStreamInput` individually to use bulk reads from the buffer because it implicitly assumed that the filter stream input wouldn't override any of the bulk operations. + +commit 81e47e9cab90d42448a13385c7a916304e90c1a1 +Author: Martijn van Groningen +Date: Mon Feb 17 12:35:07 2020 +0100 + + Improve watcher rolling upgrade tests (#52404) + + Relates to #33185 + +commit d3db6cbf5023c76f941f593fefe96527c85220c7 +Author: Martijn van Groningen +Date: Mon Feb 17 09:03:51 2020 +0100 + + Fix NPE in cluster state collector for monitoring. (#52371) + + Take into account a null license may be returned by the license service. + + Closes #52317 + +commit c9f72a0116a56ef071cd1bac76e8b577f5cebca0 +Author: Jason Tedor +Date: Sun Feb 16 17:21:28 2020 -0500 + + Fix shard follow task cleaner under security (#52347) + + The shard follow task cleaner executes on behalf of the user to clean up + a shard follow task after the follower index has been + deleted. Otherwise, these persistent tasks are left laying around, and + they fail to execute because the follower index has been deleted. In the + face of security, attempts to complete these persistent tasks would + fail. This is because these cleanups are executed under the system + context (this makes sense, they are happening on behalf of the user + after the user has executed an action) but the system role was never + granted the permission for persistent task completion. This commit + addresses this by adding this cluster privilege to the system role. + +commit 012420a495aa23157c5586ccbb4ce87ce94d38a2 +Author: Jason Tedor +Date: Sun Feb 16 17:18:08 2020 -0500 + + Add comment explaining priority of JVM options (#52348) + + Reading the startup scripts does not elucidate how JVM options are + applied. Instead, the reader must consult the source for the JVM options + parser. This commit adds some transparency around this process so that + it easier to understand reading the startup scripts how the final JVM + options to start Elasticsearch are constructed. + +commit f0747e607d23b92c00870a7392776d66847550c7 +Author: Hendrik Muhs +Date: Sun Feb 16 11:16:54 2020 +0100 + + delete the transform to delete any docs which might have been written by the (#52360) + + delete the transform to delete any docs which might have been written by the task after deleting + the index + + fixes #51347 + +commit bd3a70db4e7b092dd97130a58b7dcc979a3a232a +Author: Andrei Dan +Date: Sat Feb 15 18:42:05 2020 +0000 + + ILM fix the init step to actually be retryable (#52076) (#52375) + + We marked the `init` ILM step as retryable but our test used `waitUntil` + without an assert so we didn’t catch the fact that we were not actually + able to retry this step as our ILM state didn’t contain any information + about the policy execution (as we were in the process of initialising + it). + + This commit manually sets the current step to `init` when we’re moving + the ilm policy into the ERROR step (this enables us to successfully + move to the error step and later retry the step) + + * ShrunkenIndexCheckStep: Use correct logger + + (cherry picked from commit f78d4b3d91345a2a8fc0f48b90dd66c9959bd7ff) + Signed-off-by: Andrei Dan + +commit 0656a33d622a7d9a8b6e67b553c1fab68babd1bb +Author: Ryan Ernst +Date: Sat Feb 15 10:17:23 2020 -0800 + + Remove reference to removed bats test tasks + + The oss and default bats tests were removed, but these references to + them remained, causing gradle failures when trying to run packaging + tests. While the upgrade and plugins bats tests should still be tested, + that is being handled in #51565. This commit removes the outdated + references. + + closes #51974 + +commit 5b32d112e178539a8ed98b7148a43241cea9a0da +Author: Hicham Mallah +Date: Sat Feb 15 11:12:09 2020 +0200 + + SQL: Fix issues with GROUP BY queries (#41964) + + Translate to an agg query even if only literals are selected, + so that the correct number of rows is returned (number of buckets). + + Fix issue with key only in GROUP BY (not in select) and WHERE clause: + Resolve aggregates and groupings based on the child plan which holds + the info info for all the fields of the underlying table. + + Fixes: #41951 + Fixes: #41413 + (cherry picked from commit 45b85809678b34a448639a420b97e25436ae851f) + +commit b7291573b2ff4d1652b8362ef0c6fbad3eb3495a +Author: William Brafford +Date: Fri Feb 14 16:00:14 2020 -0500 + + Always set default ES_PATH_CONF for package scriptlets (#51827) (#52381) + + * Set default ES_PATH_CONF for package scriptlets + + Our packages use scriptlets to create or update the Elasticsearch + keystore as necessary when installing or upgrading an Elasticsearch + package. If these scriptlets don't work as expected, Elasticsearch may + try and fail to create or upgrade the keystore at startup time. This + will prevent Elasticsearch from starting up at all. + + These scriptlets use the Elasticsearch keystore command-line tools. Like + most of our command-line tools, the keystore tools will by default get + their value for ES_PATH_CONF from a system configuration file: + /etc/sysconfig/elasticsearch for RPMs, /etc/default/elasticsearch for + debian packages. Previously, if the user removed ES_PATH_CONF from that + system configuration file (perhaps thinking that it is obsolete when + the same variables is also defined in the systemd unit file), the + keystore command-line tools would fail. Scriptlet errors do not seem to + cause the installation to fail, and for RPMs the error message is easy + to miss in command output. + + This commit adds a line of bash to scriptlets that will set ES_PATH_CONF + to a default when ES_PATH_CONF is unset, rather than only when the + system configuration file is missing. + +commit 0da183339ea1e0f696fa27e6260b28fe69fbf13a +Author: OriGlassman <39296766+OriGlassman@users.noreply.github.com> +Date: Fri Feb 14 20:41:51 2020 +0200 + + [DOCS] Fixed "SeachRequest" -> "SearchRequest" typo in HLRC docs (#52144) + +commit 9128106b4c4f860c05542db8c8389549ecfd4c6a +Author: James Rodewig +Date: Fri Feb 14 12:33:21 2020 -0500 + + [DOCS] Remove 'analyzed string' references (#51946) + + The `string` field datatype was replaced by the `text` and `keyword` + field datatypes in [5.0][0]. + + This removes several outdated references to 'analyzed string' fields. + + [0]:https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking_50_mapping_changes.html#_string_fields_replaced_by_textkeyword_fields + +commit da2d441d5058d4a0147221e3f279f6d0cb4f301c +Author: Andrei Dan +Date: Fri Feb 14 17:31:24 2020 +0000 + + ILM make the set-single-node-allocation retryable (#52077) (#52138) + + (cherry picked from commit 0e473115958f691fc8dc87293642aea6a07fe3da) + Signed-off-by: Andrei Dan + +commit 4eea9c20ee2d2885f814a485b439fe68bcb2163e +Author: Andrei Stefan +Date: Fri Feb 14 18:58:45 2020 +0200 + + SQL: document the use of a filter on _routing (#52355) + + * Fix "Description"s for various sections in the functions pages. + * Added a TIP for searching using a routing key. + * Other small polishings + + (cherry picked from commit 9fad0b1ac4409a42c435ed040f41cbaea18930a3) + +commit bb932c85812f97a3f28b5c5e7899377da13f2b07 +Merge: 74e7777cbb4 e77e49e956a +Author: Andrei Stefan +Date: Fri Feb 14 18:59:45 2020 +0200 + + Merge branch '7.x' of https://github.com/elastic/elasticsearch into 7.x + +commit e77e49e956aefc7fb0475fb4af3d1ff4a4ebd154 +Author: Lisa Cawley +Date: Fri Feb 14 08:51:55 2020 -0800 + + [DOCS] Adds machine learning highlights (#52334) + +commit 2311bc62f7026a9dbc1b563475bfb7d6509c573d +Author: William Brafford +Date: Fri Feb 14 11:39:13 2020 -0500 + + Filter empty lines from "docker ls" response (#52081) + + * Filter empty lines from docker ls response + + In order to cut down on test time, our docker/vagrant tests build the + docker image outside of the vagrant VM. When we get around to launching + the Vagrant VM, we mount that already-built docker image to a known + location. At that point, we need to load the docker image. But we only + want to load it once. As we're running tests, we use "docker ls" to + check whether the local image is loaded for use. Empty output from the + particular ls invocation means no image is loaded. + + There was a bug in how we checked this. In Java, splitting an empty + string will yield an array containing one empty string. So when we're + counting the output from the docker ls command, we need to filter out + empty lines in order to proceed to loading the image for docker tests. + +commit 146def8caade3e60992679112ae1202689afc96c +Author: Nik Everett +Date: Fri Feb 14 11:19:11 2020 -0500 + + Implement top_metrics agg (#51155) (#52366) + + The `top_metrics` agg is kind of like `top_hits` but it only works on + doc values so it *should* be faster. + + At this point it is fairly limited in that it only supports a single, + numeric sort and a single, numeric metric. And it only fetches the "very + topest" document worth of metric. We plan to support returning a + configurable number of top metrics, requesting more than one metric and + more than one sort. And, eventually, non-numeric sorts and metrics. The + trick is doing those things fairly efficiently. + + Co-Authored by: Zachary Tong + +commit ad56802ac606b30a66f9e037eba0a1a2e6e32317 +Author: Dimitris Athanasiou +Date: Fri Feb 14 17:16:06 2020 +0200 + + [7.x][ML] Refactor ML mappings and templates into JSON resources (#51… (#52353) + + ML mappings and index templates have so far been created + programmatically. While this had its merits due to static typing, + there is consensus it would be clear to maintain those in json files. + In addition, we are going to adding ILM policies to these indices + and the component for a plugin to register ILM policies is + `IndexTemplateRegistry`. It expects the templates to be in resource + json files. + + For the above reasons this commit refactors ML mappings and index + templates into json resource files that are registered via + `MlIndexTemplateRegistry`. + + Backport of #51765 + +commit cabc1769e2e18a3c35b5ad10cb353ea4b3f24088 +Author: bellengao +Date: Fri Feb 14 21:30:26 2020 +0800 + + [DOC] Remove definition typo in update alias API docs (#52184) + + Removes an erroneously duplicated definition heading from the update alias API reference docs. + +commit 53b6583fedcf2d8d81e58c4e429e9c47c3aab86c +Author: Nik Everett +Date: Fri Feb 14 07:07:56 2020 -0500 + + Decode max and min optimization more carefully (#52336) (#52358) + + Fixes the the no-query optimization for `min` and `max` aggregations + for `date_nanos` fields by delegating decoding dates "through" their + `resolution` member. + + Closes #52220 + +commit 51e74be1bbd5802339cf89edd3d0b9186188c87d +Author: Marios Trivyzas +Date: Fri Feb 14 11:22:46 2020 +0100 + + SQL: [Tests] Add tests for fixed issues (#52335) + + Add tests to verify behaviour for + fixed issues: #33724 & #38306 + + (cherry picked from commit 89fb6753a9db9484a5622417cd4ffea9af0347ad) + +commit 6cd42923d5ffb51274341610d102e852d359e511 +Author: Ioannis Kakavas +Date: Fri Feb 14 10:24:45 2020 +0200 + + Update cryptacular to 1.2.4 (#52331) (#52349) + + Cryptacular is a dependency of opensaml + +commit efd7542b2afb54a6bbd9ab7ef8edfa31ade02bab +Author: Hendrik Muhs +Date: Fri Feb 14 07:48:23 2020 +0100 + + [7.x][Transform] provide exponential_avg* stats for batch transforms (#52041) (#52323) + + provide exponential_avg* stats for batch transforms, avoids confusion + why those values are all 0 otherwise + +commit a66988281f1d2860e5822165fe89c3edecfe6a18 +Author: Igor Motov +Date: Thu Feb 13 18:09:26 2020 -0500 + + Add histogram field type support to boxplot aggs (#52265) + + Add support for the histogram field type to boxplot aggs. + + Closes #52233 + Relates to #33112 + +commit c61124a7b9ce5b9faf4890b0b2eaf427d3258d1e +Author: Mark Vieira +Date: Thu Feb 13 14:23:51 2020 -0800 + + Fix input snapshotting error when using test clusters cliSetup (#52340) + +commit 0d7165a40bd7692fdc766df8d8e9902105545033 +Author: Julie Tibshirani +Date: Thu Feb 13 12:59:59 2020 -0800 + + Standardize naming of fetch subphases. (#52171) + + This commit makes the names of fetch subphases more consistent: + * Now the names end in just 'Phase', whereas before some ended in + 'FetchSubPhase'. This matches the query subphases like AggregationPhase. + * Some names include 'fetch' like FetchScorePhase to avoid ambiguity about what + they do. + +commit 0da3af7581129a67addaa0938f04bebcc5ff83c0 +Author: Przemysław Witek +Date: Thu Feb 13 16:55:47 2020 +0100 + + [7.x] [ML] Add _cat/ml/data_frame/analytics API (#52260) (#52312) + +commit ea6f0e39bc22a48e819e7d7864d96decb3cc30e7 +Author: Marios Trivyzas +Date: Thu Feb 13 15:36:31 2020 +0100 + + [Tests] Update skip version for YAML tests (#52310) + + Update skip versions upper boundary to match the release + or intented release version of the feature/fix. + +commit cdd8f38d7129ec84868a64c0baf9c1c6aed2be75 +Author: David Kyle +Date: Thu Feb 13 13:02:21 2020 +0000 + + Remove unneeded test PreventFailingBuildIT (#52198) + + Ironically PreventFailingBuildIT.testSoThatTestsDoNotFail is causing failures + as documented in #52197. The no longer serves a purpose and can now be removed. + +commit ac535f59a50e4f479d6eaf7d5acfaadef0390c44 +Author: Nik Everett +Date: Thu Feb 13 07:56:29 2020 -0500 + + Enable BWC test after backport (#52300) + + Now that we've backported #52016 we can run its tests when we're + performance backwards compatibility testing. + +commit 5373a77fb9a94915d9b2c44279d0dc4edee9a0ef +Author: Costin Leau +Date: Thu Feb 13 14:14:42 2020 +0200 + + QL: Extract common Failure class (#52281) + + Shared across SQL and EQL + + (cherry picked from commit 1aeda20d3ec3d6c885de03c6043dd1e8eab9f230) + +commit b0ad37126c42d110a3fb0b0ae95e50491fbe0586 +Author: Rory Hunter +Date: Thu Feb 13 13:18:13 2020 +0100 + + Remove setuid,setgid bits on all files in Docker image (#51851) + + This is to mitigate "stackclash" attacks. This is a a very small partial + backport from #50277. + +commit 3ea49557fe41f791e8cc6a50ad97e427b7650285 +Author: David Roberts +Date: Thu Feb 13 09:33:02 2020 +0000 + + Add cluster:admin/analyze permission to Kibana system role (#52259) + + This is to support the ML categorization wizard. + + Currently cluster:admin/analyze is only provided with the + "manage" cluster privilege, which is an excessive privilege + level to provide access to this single feature. It means + that the ML categorization wizard only works for extremely + highly privileged users. + + Following this change the Kibana system user will be + permitted to run the _analyze endpoint on supplied strings + (not on an index). The ML UI will then call the _analyze + endpoint as the Kibana system user after first checking + that the logged-in user is permitted to create an ML job. + This will mean that users with the more reasonable + "manage_ml" cluster privilege will be permitted to use + the ML categorization wizard. + + (This is also consistent with the way the ML UI will access + _all_ Elasticsearch functionality when the "ML in Spaces" + project is completed.) + + Closes #51391 + Relates elastic/kibana#57375 + +commit 291713f28402c4d0bf1b530c97796d5d95789332 +Author: debadair +Date: Wed Feb 12 17:52:11 2020 -0800 + + [DOCS] Fixed typo in jump link. (#52302) + +commit 2dac36de4d578974d26e9b5083e25c572109c892 +Author: Nik Everett +Date: Wed Feb 12 19:25:05 2020 -0500 + + HLRC support for string_stats (#52163) (#52297) + + This adds a builder and parsed results for the `string_stats` + aggregation directly to the high level rest client. Without this the + HLRC can't access the `string_stats` API without the elastic licensed + `analytics` module. + + While I'm in there this adds a few of our usual unit tests and + modernizes the parsing. + +commit 12e378b3ac9170276c152050a24bd8b15abb77a6 +Author: Ryan Ernst +Date: Wed Feb 12 15:54:39 2020 -0800 + + Fix incorrect date nanos docs example (#52249) + + The example of how to access the nano value of a date_nanos field has + been broken since it was created. This commit fixes it to use the + correct scripting methods. + + closes #51931 + +commit f0668cabbc0b96a36d666716c7dead04a9776a15 +Author: Julie Tibshirani +Date: Wed Feb 12 15:16:04 2020 -0800 + + Adjust the 'skip' version in flattened REST tests. (#52293) + + I forgot to adjust it after backporting the flattened fields feature. + +commit 7efce22f19d138d081ae2ad9516758ec7b7a0e86 +Author: Nik Everett +Date: Wed Feb 12 17:57:04 2020 -0500 + + Fix a DST error in date_histogram (backport #52016) (#52237) + + When `date_histogram` attempts to optimize itself it for a particular + time zone it checks to see if the entire shard is within the same + "transition". Most time zone transition once every size months or + thereabouts so the optimization can usually kicks in. + + *But* it crashes when you attempt feed it a time zone who's last DST + transition was before epoch. The reason for this is a little twisted: + before this patch it'd find the next and previous transitions in + milliseconds since epoch. Then it'd cast them to `Long`s and pass them + into the `DateFieldType` to check if the shard's contents were within + the range. The trouble is they are then converted to `String`s which are + *then* parsed back to `Instant`s which are then convertd to `long`s. And + the parser doesn't like most negative numbers. And everything before + epoch is negative. + + This change removes the + `long` -> `Long` -> `String` -> `Instant` -> `long` chain in favor of + passing the `long` -> `Instant` -> `long` which avoids the fairly complex + parsing code and handles a bunch of interesting edge cases around + epoch. And other edge cases around `date_nanos`. + + Closes #50265 + +commit 12cb6dcefe4dcf30abe1a301560f408abbd30f7c +Author: Nhat Nguyen +Date: Wed Feb 12 16:16:59 2020 -0500 + + Fix testFlushOnInactive (#52275) + + We need to reduce the translog sync interval for indices with translog + async setting so that we can have the safe commit in the assertBusy + interval. This is needed since #51905, where we use the local checkpoint + of the safe commit to calculate the number of uncommitted operations of + a translog stats. + + Closes #52251 + Relates #51905 + +commit 5bcc6fce5cc7d6381f44eabb92c9ac8e2b9cddba +Author: Jay Modi +Date: Wed Feb 12 15:05:41 2020 -0700 + + Remove DeprecationLogger from route objects (#52285) + + This commit removes the need for DeprecatedRoute and ReplacedRoute to + have an instance of a DeprecationLogger. Instead the RestController now + has a DeprecationLogger that will be used for all deprecated and + replaced route messages. + + Relates #51950 + Backport of #52278 + +commit 8c930a9960b46da015b26afcf0b434502887ba90 +Author: Nik Everett +Date: Wed Feb 12 17:01:27 2020 -0500 + + Update skip after backport (#52288) + + Now that #51868 is fully backported we can run its tests in the + backwards compatibility tests. + +commit dac720d7a16201f3339866352f0a601794da2d5d +Author: Marios Trivyzas +Date: Wed Feb 12 22:56:14 2020 +0100 + + Add a cluster setting to disallow expensive queries (#51385) (#52279) + + Add a new cluster setting `search.allow_expensive_queries` which by + default is `true`. If set to `false`, certain queries that have + usually slow performance cannot be executed and an error message + is returned. + + - Queries that need to do linear scans to identify matches: + - Script queries + - Queries that have a high up-front cost: + - Fuzzy queries + - Regexp queries + - Prefix queries (without index_prefixes enabled + - Wildcard queries + - Range queries on text and keyword fields + - Joining queries + - HasParent queries + - HasChild queries + - ParentId queries + - Nested queries + - Queries on deprecated 6.x geo shapes (using PrefixTree implementation) + - Queries that may have a high per-document cost: + - Script score queries + - Percolate queries + + Closes: #29050 + (cherry picked from commit a8b39ed842c7770bd9275958c9f747502fd9a3ea) + +commit 40b58e612db1f4ca3670d46b96c636606db9639b +Author: Lisa Cawley +Date: Wed Feb 12 13:43:21 2020 -0800 + + [DOCS] Fixes, sorts ML tagged regions (#52283) + +commit d9fd6fc90c5204cbd0478ceea33cd453f0c33933 +Author: Marios Trivyzas +Date: Wed Feb 12 21:50:57 2020 +0100 + + SQL: [Docs] Fix typo + + Add missing closing "`" + + Follows: c2e055253777748a3680f4abaa846d8162adb0d7 + +commit 0c1889389ad01be6323d003c651728964ffd8a7a +Author: Nik Everett +Date: Wed Feb 12 13:55:47 2020 -0500 + + Update skip for backported fix (#52241) + + Now that #51172 is fully backported we can fix the `skip` clause in the + bwc tests for it. + +commit c07f46409cdd246fcfadbf2cef77c44e3342938a +Author: Ryan Ernst +Date: Wed Feb 12 10:48:15 2020 -0800 + + Fix single newline in logging output stream buffer (#52253) + + The buffer in LoggingOutputStream skips flushing when only a newline + appears. However, if a windows newline appeared, the buffer length was + not reset. This commit resets the length so the \r does not appear in + the next logging message. + + closes #51838 + +commit 5dfe27601ecbd62314abb91ed1b7d75948171454 +Author: Bogdan Pintea +Date: Wed Feb 12 19:45:12 2020 +0100 + + SQL: supplement input checks on received request parameters (#52229) (#52277) + + * Add more checks around parameter conversions + + This commit adds two necessary verifications on received parameters: + - it checks the validity of the parameter's data type: if the declared + data type is resolved to an ES or Java type; + - it checks if the returned converter is non-null (i.e. a conversion is + possible) and generates an appropriate exception otherwise. + + (cherry picked from commit eda30ac9c69383165324328c599ace39ac064342) + +commit fc964643bdc123295a7ede9cb11e33c17ae23258 +Author: James Rodewig +Date: Wed Feb 12 12:59:24 2020 -0500 + + [DOCS] Add docs build info to TESTING.asciidoc (#52271) + + Adds a brief section about Elasticsearch docs and how users can + test/build them locally. + +commit 6ea3f5ada126eb97a2f9a1b7c3f60fb98ab2e75a +Author: Armin Braun +Date: Wed Feb 12 18:35:50 2020 +0100 + + Move EC2 Discovery Tests to Mock Rest API (#50605) (#52270) + + Move EC2 discovery tests to using the mock REST API introduced in + https://github.com/elastic/elasticsearch/pull/50550 instead of mocking + the AWS SDK classes manually. + Move the trivial remaining AWS SDK mocks to the single test suit that + was using them. + +commit 26900bfb0563a8f7b5e26fdbd68e27c093ec1cf9 +Author: Costin Leau +Date: Mon Feb 10 18:19:24 2020 +0200 + + EQL: Add infra for planning and query folding (#52065) + + Actual folding not yet in place (TBD) + + (cherry picked from commit d52b96f273a94c90e475a5035cd57baa086fb0c0) + +commit e098e837f713ce9ac83ed5c1b1319e1de2833462 +Author: Nhat Nguyen +Date: Wed Feb 12 11:08:14 2020 -0500 + + Fix testShouldPeriodicallyFlushAfterMerge (#52243) + + MockRandomMergePolicy randomly determines if a segment should use a + compound format. This can cause a force merge performing two merges: (1) + merging to a single segment, (2) rewriting the new segment using the + compound format. If the second merge completes after we have flushed, + then it can flip the flag shouldPeriodicallyFlushAfterBigMerge to true. + + Closes #52205 + +commit 257eb0212c64c2d12e233e1d4ffa9a27f1b556ed +Author: Nhat Nguyen +Date: Wed Feb 12 10:25:28 2020 -0500 + + Mute ‘test user agent processor with non-ECS schema’ + + Tracked at #52266 + +commit ca34817659734778d888c8f2d2a96a73dfac8e92 +Author: James Rodewig +Date: Wed Feb 12 08:45:15 2020 -0500 + + [DOCS] Add EQL limitations page (#52001) + + Documents limitations for EQL in Elasticsearch. + +commit 20453d3ac842dd7ab427d8a7757d5233cb662a62 +Author: James Rodewig +Date: Wed Feb 12 08:40:10 2020 -0500 + + [DOCS] Add basic EQL search tutorial docs (#51574) + + I plan to add additional sections to this page with future PRs: + + * Specify timestamp and event type fields + * Specify a join key field + * Filter using query DSL + * Paginate a large response + + See #51057. + +commit 5d35eaa1cbe008aa18096c1c607c257599d4fc6c +Author: Hendrik Muhs +Date: Wed Feb 12 14:14:36 2020 +0100 + + [Transform] improve irrecoverable error detection - part 2 (#52003) + + base error handling on rest status instead of listing individual exception types + + relates to #51820 + +commit 3f151d1d75c50dbef28ffb09c266e3a6c9ce802e +Author: James Rodewig +Date: Fri Jan 31 10:35:04 2020 -0500 + + [DOCS] Add redirects, update JSON spec to fix docs build (#51747) + + Docs build [#11556][0] broke due to several outdated or incorrect links + in the JSON REST spec. + + This fixes those links where possible and adds redirects. + + [0]: https://elasticsearch-ci.elastic.co/job/elastic+docs+master+build/11556/ + +commit c2e055253777748a3680f4abaa846d8162adb0d7 +Author: Marios Trivyzas +Date: Wed Feb 12 12:54:42 2020 +0100 + + SQL: [Docs] Add limitation for sorting on aggs (#52210) + + Add a section to point out that when ordering by an aggregate + only plain aggregate functions are allowed, no scalars/operators + can be used on top of them. + + Fixes: #52204 + (cherry picked from commit 78a1185549ff7f3229fd2d036567eb2a4f2cf230) + +commit a3ebacfcf3ca89fd18f9f2d08951286facbdb178 +Author: Andrei Stefan +Date: Wed Feb 12 11:20:03 2020 +0200 + + 52169 & 52172 7x backport (#52256) + + * Extract common optimizer tests (#52169) + + (cherry picked from commit e5ad72bc22e9ec0686ab582195f0032efcb880bf) + + * Hook in the optimizer rules (#52172) + + (cherry picked from commit 1f90d8cc56052fbf2af604e72f9f5ca73f5e75d5) + +commit daab242c7582f51319acaf065b293cae8831c0ff +Author: Marios Trivyzas +Date: Wed Feb 12 09:37:36 2020 +0100 + + SQL: Fix ORDER BY on aggregates and GROUPed BY fields (#51894) + + Previously, in the in-memory sorting module + `LocalAggregationSorterListener` only the aggregate functions where used + (grabbed by the `sortingColumns`). As a consequence, if the ORDER BY + was also using columns of the GROUP BY clause, (especially in the case + of higher priority - before the aggregate functions) wrong results were + produced. E.g.: + ``` + SELECT gender, MAX(salary) AS max FROM test_emp + GROUP BY gender + ORDER BY gender, max + ``` + + Add all columns of the ORDER BY to the `sortingColumns` so that the + `LocalAggregationSorterListener` can use the correct comparators in + the underlying PriorityQueue used to implement the in-memory sorting. + + Fixes: #50355 + (cherry picked from commit be680af11c823292c2d115bff01658f7b75abd76) + +commit 74e7777cbb414444e6eda803fc31aa2a4efe71ad +Author: Andrei Stefan +Date: Wed Feb 12 06:57:43 2020 +0200 + + Hook in the optimizer rules (#52172) + + (cherry picked from commit 1f90d8cc56052fbf2af604e72f9f5ca73f5e75d5) + +commit a21e2b211a3af10de48c69d737f8e92ffffaa2d1 +Author: Andrei Stefan +Date: Wed Feb 12 06:57:12 2020 +0200 + + Extract common optimizer tests (#52169) + + (cherry picked from commit e5ad72bc22e9ec0686ab582195f0032efcb880bf) + +commit edaf6d1f7968685765fbf648288be4d9220c9faf +Author: Hendrik Muhs +Date: Wed Feb 12 07:48:04 2020 +0100 + + [Transform] maintain a list of unsupported aggregations in transforms (#52190) (#52222) + + add a list of unsupported aggs in transforms and create a test that fails if a new aggregation is + added. Limitation: works only if a new agg is added to either the core or a known plugin + (Analytics, MatrixAggregation). + +commit dd14210689b55b813cbb656bc5bba58cf796fa7f +Author: Lisa Cawley +Date: Tue Feb 11 18:27:33 2020 -0800 + + [DOCS] Clarifies machine learning built-in roles (#51504) + +commit 79e5e809b6c6a5c9088202acc9d3afee2e2dc4d2 +Author: Jason Tedor +Date: Tue Feb 11 20:54:40 2020 -0500 + + Add unit tests for reading JVM options files (#52176) + + This commit adds some unit tests to cover the reading of JVM options + files. + +commit 2a968f4f2b49092baa271ffa8a3749241008325d +Author: Benjamin Trent +Date: Tue Feb 11 17:54:44 2020 -0500 + + [ML] job results provider refactoring (#52012) (#52238) + + During a bug hunt, I caught a handful of things (unrelated to the bug) that could be potential issues: + + 1. Needlessly wrapping in exception handling (minor cleanup) + 2. Potential of notifying listeners of a failure multiple times + even trying to notify of a success after a failure notification + +commit 28c56da75495bcd94f35a646d8f67f9ef97b28a5 +Author: Mark Vieira +Date: Tue Feb 11 13:32:59 2020 -0800 + + Don't track absolute path as test input to improve cacheability (#52235) + +commit d48ce12920e66c03d159bcdd0d8432294c11423c +Author: Gordon Brown +Date: Tue Feb 11 14:18:55 2020 -0700 + + Convert ILM and SLM histories into hidden indices (#51456) + + Modifies SLM's and ILM's history indices to be hidden indices for added + protection against accidental querying and deletion, and improves + IndexTemplateRegistry to handle upgrading index templates. + + Also modifies the REST test cleanup to delete hidden indices. + +commit bb2e04bc16377cdfbb8927c708199fd5541a2684 +Author: Jason Tedor +Date: Tue Feb 11 15:07:59 2020 -0500 + + Use absolute path for temporary directory in tests (#52228) + + We explicitly set the path for the temporary directory to use in test + tasks, but today this path is a relative path, relative to the current + working directory of the test task. The fact that we are using a + relative path here appears to be legacy, simply leftover from the days + of the Maven build. An absolute path is preferred here, since it's + explicit and we do not have to rely on everyone resolving the path + properly relative to the working directory. + +commit 6ed3311443a803e03bd5b4f6d107dc827f9269fe +Author: Jason Tedor +Date: Tue Feb 11 14:52:34 2020 -0500 + + Ensure test temporary directory exists (#52227) + + Today we we set the test temporary directory explicitly by controling + java.io.tmpdir. Yet, we do not guarantee this directory exists, instead + relying on a test base class (LuceneTestCase) to create this directory + when it initializes. However, some of our tests do not rely on our test + framework, and thus do not have access to LuceneTestCase, instead + relying on RandomizedRunner directly. We should not be relying on the + temporary directory being implicitly created, instead guaranteeing that + it exists before test execution starts. This commit does that by + creating the test temporary directory before the test task executes (via + a doFirst). + +commit 0372d6d239c03f215d262286cdae5bfd769dabaa +Author: Zachary Tong +Date: Tue Feb 11 12:47:03 2020 -0500 + + Allow ObjectParsers to specify required sets of fields (#49661) + + ConstructingObjectParser can be used to specify required fields, + but it is still difficult to configure "sets" of fields where only + one of the set is required (requiring hand-rolled logic in each + ConstructingObjectParser, or adding special validation methods + to objects that are called after building the object). + + This commit adds a new method on ObjectParser which allows + the parsers to register required sets. E.g. ["foo", "bar"] can be + registered, which means "foo", "bar" or both must be configured + by the user otherwise an exception is thrown. + + This pattern crops up in many places in our parsers; a good example are + the aggregation "field" and "script" fields. One or both must be + configured on all aggregations, omitting both should result in an exception. + This was previously handled far downstream resulting in an aggregation + exception, when it should be a parse exception. + +commit 86d5211c059bb0109281f2a904cbffc154ff62d5 +Author: Nik Everett +Date: Tue Feb 11 12:58:40 2020 -0500 + + Make sorting by an agg results a real abstraction (#52007) (#52212) + + This removes a bunch of `instanceof`s in favor of two new methods on + `InernalAggregation`. The default implementations of these methods just + throw exceptions explaining that you can't sort on this aggregation. + They are overridden by all of the classes that used to have `instanceof` + checks against them. + + I doubt this is really any faster in practice. The real benefit here is + that it is a little more obvious *that* you can sort by the results of + an aggregation and it should be *much* more obvious where to look at + *how* aggregations sort themselves. + + There are still a bunch more `instanceof`s in left in `AggregationPath` + but those will wait for a followup change. + +commit cc1fce96bad49d3d9b8e0921a992631f1271c945 +Author: Albert Zaharovits +Date: Tue Feb 11 19:58:06 2020 +0200 + + Add a new async search security origin (#52141) + + This commit adds a new security origin, and an associated reserved user + and role, named `_async_search`, which can be used by internal clients to + manage the `.async-search-*` restricted index namespace. + +commit d68a4ec82e69f2be23e95a58759b8bd956f04373 +Author: James Rodewig +Date: Tue Feb 11 11:49:49 2020 -0500 + + [7.x] Permit EQL feature flag in release builds (#52201) (#52214) + + 7.x backport of #52201 + + Provides a path to set register the EQL feature flag in release builds. + This enables EQL in release builds so that release docs tests pass. + + Release docs tests do not have infrastructure in place to only register + snippets from included portions of the docs, they instead include all + docs snippets. + + Since EQL can not be enabled in release builds, this meant that the EQL + snippets fail in the release docs tests. + + This adds the ability to enable EQL in the release docs tests. This + system property will be removed when EQL is ready for release. + +commit 098380e4830c29ba22107678164579f7227b1529 +Author: Hendrik Muhs +Date: Tue Feb 11 17:24:15 2020 +0100 + + Percentiles aggregation validation checks for range (#51871) + + disallow to specify percentile out of range [0,100]. This also fixes a problem in transform by failing + validation if an invalid percentile configuration is used. + +commit 6fe8f1649b06207df2afb5b33c22914d80bc6158 +Author: James Rodewig +Date: Thu Feb 6 14:44:20 2020 -0500 + + [DOCS] Include docs on permanently unreleased branches only (#51743) + + Adds the ability to display docs on permanently unreleased branches, + such as `master` and `7.x`. + + Also updates how the autoscaling and EQL docs are included. + Currently, these feature-flag docs would display on any unreleased + branches that contain the changes, such as 7.7. + +commit d1d9c40e718182a66fdc26502c790a052781fbb1 +Author: David Roberts +Date: Tue Feb 11 15:26:59 2020 +0000 + + [ML] Switch poor categorization audit warning to use status field (#52195) + + In #51146 a rudimentary check for poor categorization was added to + 7.6. + + This change replaces that warning based on a Java-side check with + a new one based on the categorization_status field that the ML C++ + sets. categorization_status was added in 7.7 and above by #51879, + so this new warning based on more advanced conditions will also be + in 7.7 and above. + + Closes #50749 + +commit 473468d7630e80a7ccb6d2974f4e65fbd2f5294f +Author: David Roberts +Date: Tue Feb 11 15:22:21 2020 +0000 + + [ML] Better error when persistent task assignment disabled (#52014) + + Changes the misleading error message when attempting to open + a job while the "cluster.persistent_tasks.allocation.enable" + setting is set to "none" to a clearer message that names the + setting. + + Closes #51956 + +commit 87854573e4ce28c88d53c6599fec9edec379a173 +Author: Zachary Tong +Date: Tue Feb 11 09:38:10 2020 -0500 + + Add version constant for 7.6.1 + +commit 667e1a522575f35053ef35064fe7ae1e241cd126 +Author: Igor Motov +Date: Tue Feb 11 09:38:17 2020 -0500 + + Add Boxplot Aggregation (#52174) + + Adds a `boxplot` aggregation that calculates min, max, medium and the first + and the third quartiles of the given data set. + + Closes #33112 + +commit 204d086266f5e130b944a753cad1cb6044b47ecb +Author: Marios Trivyzas +Date: Tue Feb 11 14:59:06 2020 +0100 + + SQL: Fix issue with timezone when paginating (#52101) + + Previously, when the specified (or default) fetchSize led to + subsequent HTTP requests and the usage of cursors, those subsequent + were no longer using the client timezone specified in the initial + SQL query. As a consequence, Even though the query is executed once + (with the correct timezone) the processing of the query results by + the HitExtractors in the next pages was done using the default + timezone Z. This could lead to incorrect results. + + Fix the issue by correctly using the initially specified timezone, + which is found in the deserialisation of the cursor string. + + Fixes: #51258 + (cherry picked from commit 8f7afbdeb9295999b48a6c36db5b31cbe0cee432) + +commit 00b9098250295b2ff775f07563f38b7a2eaee4e2 +Author: David Turner +Date: Tue Feb 11 14:00:06 2020 +0000 + + Ignore timeouts with single-node discovery (#52159) + + Today we use `cluster.join.timeout` to prevent nodes from waiting indefinitely + if joining a faulty master that is too slow to respond, and + `cluster.publish.timeout` to allow a faulty master to detect that it is unable + to publish its cluster state updates in a timely fashion. If these timeouts + occur then the node restarts the discovery process in an attempt to find a + healthier master. + + In the special case of `discovery.type: single-node` there is no point in + looking for another healthier master since the single node in the cluster is + all we've got. This commit suppresses these timeouts and instead lets the node + wait for joins and publications to succeed no matter how long this might take. + +commit 4c88996cd79e8662facb7a993e94469006e78d88 +Author: David Roberts +Date: Tue Feb 11 12:54:09 2020 +0000 + + [DOCS] Correct important note for xpack.transform.enabled (#52194) + + Because transforms get assigned to an arbitrary data node it + is important that the transforms plugin is enabled on every + data node. + +commit 16ba59e9d15fabef78a33ff9ecf289cbd7221dcb +Author: Yang Wang +Date: Tue Feb 11 23:05:01 2020 +1100 + + Expose more authentication info to ingest pipeline (#51305) (#52119) + + The changes add more granularity for identiying the data ingestion user. + The ingest pipeline can now be configure to record authentication realm and + type. It can also record API key name and ID when one is in use. + This improves traceability when data are being ingested from multiple agents + and will become more relevant with the incoming support of required + pipelines (#46847) + + Resolves: #49106 + +commit 343ced42be4daa544807e88e93231fe3b2b3d618 +Author: David Kyle +Date: Tue Feb 11 11:43:34 2020 +0000 + + Mute LoggingOutputStreamTests.testMaxBuffer (#52193) + + Relates to https://github.com/elastic/elasticsearch/issues/51838 + +commit b0b1b133115bb9c55264d82226e869c4a31dda03 +Author: Tim Vernum +Date: Tue Feb 11 20:59:06 2020 +1100 + + Extract class to store Authentication in context (#52183) + + This change extracts the code that previously existed in the + "Authentication" class that was responsible for reading and writing + authentication objects to/from the ThreadContext. + + This is needed to support multiple authentication objects under + separate keys. + + This refactoring highlighted that there were a large number of places + where we extracted the Authentication/User objects from the thread + context, in a variety of ways. These have been consolidated to rely on + the SecurityContext object. + + Backport of: #52032 + +commit 6086fadf003e9d28e89ab5868c07b12fe5c8ab0f +Author: Dimitris Athanasiou +Date: Tue Feb 11 11:18:45 2020 +0200 + + [7.x][ML] Prepare to hold additional stats in DF Analytics task (#52134) (#52187) + + Refactors `DataFrameAnalyticsTask` to hold a `StatsHolder` object. + That just has a `ProgressTracker` for now but this is paving the + way to add additional stats like memory usage, analysis stats, etc. + + Backport #52134 + +commit c14e4666dfdba6f0f086de03d5f3759ec8f705fb +Author: Martijn van Groningen +Date: Tue Feb 11 09:39:20 2020 +0100 + + Wait for watcher to be started prior to rolling upgrade tests. (#52186) + + Backport: #52139 + + In the rolling upgrade tests, watcher is manually executed, + in rare scenarios this happens before watcher is started, + resulting in the manual execution to fail. + + Relates to #33185 + +commit cbebc26f50f13bc9870a183dfb3064a33e1e2bc4 +Author: Dimitris Athanasiou +Date: Tue Feb 11 09:55:00 2020 +0200 + + [7.x][ML] Retry persisting DF Analytics results (#52048) (#52160) + + Employs `ResultsPersisterService` from `DataFrameRowsJoiner` in order + to add retries when a data frame analytics job is persisting the results + to the destination data frame. + + Backport of #52048 + +commit 2f1631d9d0ae2f722f14f0bf8158ef7191b084c0 +Author: Andrei Stefan +Date: Tue Feb 11 09:15:47 2020 +0200 + + Telemetry data initial implementation (#51715) (#52175) + + (cherry picked from commit f1d1cceacaacf226fcd2459f34689843b822fe4b) + +commit c4525f8ccad60c54f70a5a4474abf81d7ededaab +Author: Lisa Cawley +Date: Mon Feb 10 18:06:01 2020 -0800 + + [DOCS] Adds ml-cpp PRs to release notes (#52158) + + Co-Authored-By: David Roberts + +commit 91d0996e0843490eb369549db28a8ffa832da7c7 +Author: Jason Tedor +Date: Mon Feb 10 20:21:32 2020 -0500 + + Remove unnecessary method in JvmOptionsParser (#52173) + + Back when the distribution launchers were compiled to target JDK 7, we + did not have access to the String#join method to space-delimit JVM + options. Since the launchers now target the same minimum JDK as + Elasticsearch itself, we now have access to this method and can replace + the use of spaceDelimitJvmOptions with String#join. This commit does + that. + +commit 350288ddf833560c517e26344bc2c5d4a11f090d +Author: Gordon Brown +Date: Mon Feb 10 17:01:59 2020 -0700 + + Check dot-index rules after template application (#52087) + + Previously, the dot-index rules (namely, that indices with dot-prefixed + names should be either hidden indices or system indices) was done + before* template application, and so only checked for the `index.hidden` + setting in the request, ignoring if that setting was set via a template. + + This commit moves that check to a different method, which is applied + after templates have been resolved and applied to the index settings. + +commit a99b311e2fb8f0bfde442c5cbbcec7e8417f494b +Author: Jason Tedor +Date: Mon Feb 10 18:36:31 2020 -0500 + + Refactor JvmOptionsParser for testability (#52102) + + This commit prepares the JvmOptionsParser to be more unit testable by + refactoring the class to have some input that it pulls from external + sources passed in as arguments. We do not change any functionality in + this commit, nor add any unit tests, we are only preparing the way. + +commit 5a72b237166085f8dc5075e49b02663fbaa74fdb +Author: Ryan Ernst +Date: Mon Feb 10 17:41:33 2020 -0500 + + Migrate SysV init tests from bats to java packaging (#51077) (#51498) + + This commit converts the sysv init tests from bats tests into the java + packaging tests. Since it is the last oss specific test, the bats oss + test task is also removed. + + relates #46005 + +commit 42610c6d744006c0be30b15fd034ec2b78ce5684 +Author: Mark Vieira +Date: Mon Feb 10 13:49:44 2020 -0800 + + Only pull docker images for fixture projects (#52157) + +commit 88cf8ac0a84890bd51b602282455f0c1d32b43b0 +Author: Ryan Ernst +Date: Mon Feb 10 16:29:29 2020 -0500 + + Fix windows empty line in logging capture (#52162) + + This commit fixes another edge case in handling windows newlines in our + capture of stdout/stderr to log4j. The case is that the \r appears at + the beginning of the buffer when flushing, which would unintentionally + be emitted as an empty string. This commit skips the flush if only a \r + was found. + + closes #51838 + +commit 6b600855a96c23b4591da75edd84f7078db263bd +Author: Marios Trivyzas +Date: Mon Feb 10 21:45:06 2020 +0100 + + SQL: Make parsing of date more lenient (#52137) + + Make the parsing of date more lenient + + - as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}` + - cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)` + + Closes: #49379 + (cherry picked from commit 5863b27500d5e7f6cdd8c6c62b09b84e53ca724a) + +commit 47255c4fd75105e0704a88b87201d45256cd53fb +Author: Mark Vieira +Date: Mon Feb 10 11:16:15 2020 -0800 + + Remove unnecessary CI configuration files + + Signed-off-by: Mark Vieira + +commit 28a8db730f4a5b22ee2b0bfe3fe64079df7ac8df +Author: Julie Tibshirani +Date: Mon Feb 10 10:40:11 2020 -0800 + + In FieldTypeLookup, factor out flat object field logic. (#52091) + + Currently, the logic for looking up `flattened` field types lives in the + top-level `FieldTypeLookup`. This PR moves it into a dedicated class + `DynamicKeyFieldTypeLookup`. + +commit 7b58ed0dd7ba1f3d77fbaf9edffd4b4b907ce9eb +Author: Bogdan Pintea +Date: Mon Feb 10 19:24:26 2020 +0100 + + Fix milliseconds handling in intervals (#51675) (#52156) + + This fixes: + + - the parsing of milliseconds in intervals: everything past the . used to be converted as-is to milliseconds, with no normalisation of the unit; thus, a value of .23 ended up as 23 millis in the interval, instead of 230. + - the printing of a trailing .0, in case the interval lacks the fractional part; + - tests generating a random millisecond value used to simply print it in the string about to be evaluated without a necessary front-filling of 0[s], where the amount was below 100/10. + + (The combination of first and last issues above, plus statistical "luck" made the incorrect handling pass the tests.) + + (cherry picked from commit 4de8c64f63ee37c1bcfdb9b9d3a07d09be243222) + +commit d188dda7ebba3d44bc1905f55782875fd3f81927 +Author: Jason Tedor +Date: Mon Feb 10 13:12:13 2020 -0500 + + Move docker-compose logging statement to debug (#52107) + + When docker-compose is required for a test fixture but is not + available, we warn log a message to this effect. This ends up being + noise during configuration, especially when working locally. This + commit changes the logging level of these messages to debug. + +commit 610f6814da9278025a71f429d7c1669e6397ed87 +Author: William Brafford +Date: Mon Feb 10 11:05:36 2020 -0500 + + Remove unnecessary dirname command (#51968) (#52089) + + The elasticsearch-env script changes the working directory to ES_HOME, + so we can just use bin/elasticsearch-keystore to invoke the keystore. + +commit 37a2e9bac6bbe64950609550e14268d53bb0c5b4 +Author: Lee Hinman +Date: Mon Feb 10 08:54:49 2020 -0700 + + [7.x] Allow forcemerge in the hot phase for ILM policies (#520… (#52083) + + * Allow forcemerge in the hot phase for ILM policies + + This commit changes the `forcemerge` action to also be allowed in the `hot` phase for policies. The + forcemerge will occur after a rollover, and allows users to take advantage of higher disk speeds for + performing the force merge (on a separate node type, for example). + + On caveat with this is that a `forcemerge` in the `hot` phase *MUST* be accompanied by a `rollover` + action. ILM validates policies to ensure this is the case. + + Resolves #43165 + + * Use anyMatch instead of findAny in validation + + * Make randomTimeseriesLifecyclePolicy single-pass + +commit d8169e5fdcfbc804659cdad119a783be87ff386a +Author: Armin Braun +Date: Mon Feb 10 16:50:09 2020 +0100 + + Don't Upload Redundant Shard Files (#51729) (#52147) + + Segment(s) info blobs are already stored with their full content + in the "hash" field in the shard snapshot metadata as long as they are + smaller than 1MB. We can make use of this fact and never upload them + physically to the repo. + This saves a non-trivial number of uploads and downloads when restoring + and might also lower the latency of searchable snapshots since they can save + phyiscally loading this information as well. + +commit c7cc383d33054d794a58f1f3dccebf3105f11a8c +Author: Przemysław Witek +Date: Mon Feb 10 16:32:34 2020 +0100 + + [7.x] Update persistent state document in the index the document belongs to (#51751) (#52145) + +commit c77b80f01e142ee7aff6440163c791729d50866a +Author: Martijn van Groningen +Date: Mon Feb 10 15:13:32 2020 +0100 + + Unmute smoke test monitoring with watcher. (#52140) + + Backport of #51490 + +commit 864e9d875d43a1bcc1c23427acc25dded1330b7f +Author: Nhat Nguyen +Date: Fri Feb 7 18:57:52 2020 -0500 + + Bubble up exception in follow task in ccr tests (#52085) + + It's perfectly fine if a bulk request on the follower hits + IndexShardClosedException in some CCR tests because we sometimes + close some follower shards while the follow-task is replicating operations. + Instead of failing the test immediately, this commit bubbles up that + failure to the shard follow task. + + Closes #52052 + +commit 27265f032a69a150f4696b9bba8e6b1c32b63b01 +Author: Marios Trivyzas +Date: Mon Feb 10 10:39:45 2020 +0100 + + SQL: Enhance timestamp escaped literal parsing (#52097) + + Allow also whitespace ` ` (together with `T`) as a separator between + date and time parts of the timestamp string. E.g.: + ``` + {ts '2020-02-08 12.10.45'} + ``` + or + ``` + {ts '2020-02-08T12.10.45'} + ``` + + Fixes: #46069 + (cherry picked from commit 07c977023fb8ceab5991c359a6cbfe07beaad9bb) + +commit 4e4815355a8bdcb78afdfd61033abd2dc3948161 +Author: Tim Vernum +Date: Mon Feb 10 21:19:18 2020 +1100 + + Mute DocumentSubsetBitsetCacheTests.testCacheUnderConcurrentAccess (#52135) + + Test does not always complete in expected time. + + Relates: #51914 + Backport of: #52122 + +commit fa4dcd50d9d284fad4cbe314286e1fddbc28e57b +Author: Andrei Stefan +Date: Mon Feb 10 11:48:45 2020 +0200 + + Extract common optimization rules for QL (#52054) (#52132) + + (cherry picked from commit ee43115531234c2d955193ce0c9c268e1f02ab43) + +commit 80e3c972100468325870319a952a46ad3ad3ed30 +Author: Ignacio Vera +Date: Mon Feb 10 10:13:22 2020 +0100 + + Upgrade to lucene-8.5.0-snapshot-d62f6307658 (#52039) (#52130) + +commit 1cefafdd142bf4805ccaa880b8cb60a1464588ce +Author: David Roberts +Date: Mon Feb 10 09:10:50 2020 +0000 + + [ML] Add new categorization stats to model_size_stats (#52009) + + This change adds support for the following new model_size_stats + fields: + + - categorized_doc_count + - total_category_count + - frequent_category_count + - rare_category_count + - dead_category_count + - categorization_status + + Backport of #51879 + +commit 9b7e688f5b023af399819b9600c0a7a6908723a9 +Author: Alan Woodward +Date: Fri Feb 7 19:01:21 2020 +0000 + + Don't use a static QueryShardResult for a null instance (#52063) + + Fixes #52042 + +commit 343fb36c7f01d27f507776efab01a085ecb5bf9d +Author: Ioannis Kakavas +Date: Mon Feb 10 10:57:03 2020 +0200 + + Test modifications for FIPS 140 mode (#51832) (#52128) + + - Enable SunJGSS provider for Kerberos tests + - Handle the fact that in the decrypt method in KeyStoreWrapper might + not throw immediately when the GCM cipher is from BouncyCastle FIPS + and we end up with a DataInputStream that has reached it's end. + - Disable tests, jarHell, testingConventions for ingest attachment + plugin. We don't support this plugin (and document this) in FIPS + mode. + - Don't attempt to install ingest-attachment in smoke-test-plugins + +commit 44ea1efd26b973756138b49ec8463ae27b0f0f55 +Author: Martijn van Groningen +Date: Mon Feb 10 09:23:08 2020 +0100 + + Tidy up GetSourceRequest class: (#51916) + + * No need to implement ToXContentObject + * Made index and id fields immutable. + +commit 3edadfefd0954d3a531d8307a05354d2f4468952 +Author: Jay Modi +Date: Sun Feb 9 22:48:32 2020 -0700 + + RestHandlers declare handled routes (#52123) + + This commit changes how RestHandlers are registered with the + RestController so that a RestHandler no longer needs to register itself + with the RestController. Instead the RestHandler interface has new + methods which when called provide information about the routes + (method and path combinations) that are handled by the handler + including any deprecated and/or replaced combinations. + + This change also makes the publication of RestHandlers safe since they + no longer publish a reference to themselves within their constructors. + + Closes #51622 + + Co-authored-by: Jason Tedor + + Backport of #51950 + +commit 8c0b49cd328abd935124b178187b4aca55619b07 +Author: Ioannis Kakavas +Date: Mon Feb 10 07:38:59 2020 +0200 + + Adjust jarHell and 3rd party audit exclusions (#51733) (#51766) + + Now that the FIPS 140 security provider is simply a test dependency + we don't need the thirdPartyAudit exceptions, but plugin-cli and + transport-netty4 do need jarHell disabled as they use the non fips + BouncyCastle security provider as a test dependency too. + +commit dc143d59c8499dc678128513139b572e736b13d3 +Author: Nhat Nguyen +Date: Fri Feb 7 11:09:40 2020 -0500 + + Increase shard inactive time to 1h in upgrade tests (#52051) + + Similar to the fix in #51651, this commit increases the shard inactive + timeout for x-pack. + + Closes #52031 + +commit 6d0a0e124072751f7eb74909af0d4908ffe02124 +Author: Nhat Nguyen +Date: Sun Feb 9 22:04:55 2020 -0500 + + Revert "Mute ReindexFailureTests test" + + This reverts commit 16afbf91bb8aeb22933d21fe1012e2332468977c. + + The issue was fixed in #52099 + +commit 80a9a08b057e688690f05638c9fd336e6ce8ea34 +Author: Nhat Nguyen +Date: Sun Feb 9 22:03:57 2020 -0500 + + Fix leaking searcher when shards are removed or relocated (#52099) + + We might leak a searcher if the target shard is removed (i.e., its index + is deleted) or relocated while we are creating a SearchContext from a + SearchRewriteContext. + + Relates #51708 + Closes #52021 + + I labelled this non-issue for an unreleased bug introduced in #51708. + +commit 79f67e79cf1703b91e0101c08e3dde78612db442 +Author: Tim Vernum +Date: Mon Feb 10 12:37:32 2020 +1100 + + Mute MixedCluster 180_locale_dependent_mapping (#52116) + + Muting this test as it has frequent failures. + + See: #49719 + +commit d5c015062d7b6977034b4b5ca86b189130d63442 +Author: Tim Vernum +Date: Mon Feb 10 12:23:56 2020 +1100 + + Don't allow null User.principal (#52049) + + Some parts of the User class (e.g. equals/hashCode) assumed that + principal could never be null, but the constructor didn't enforce + that. + + This adds a null check into the constructor and fixes a few tests that + relied on being able to pass in null usernames. + + Backport of: #51988 + +commit 2b9929118729c50c77035a4108a5c2b8629c6483 +Author: Jason Tedor +Date: Sun Feb 9 15:48:09 2020 -0500 + + Add autoscaling feature flag in release REST tests (#52096) + + The REST tests for autoscaling either need to be skipped in a + non-snapshot build, or alternatively, the feature flag registered so + that autoscaling can be enabled. We prefer the latter approach, as it + allows us to also test autoscaling in non-snapshot builds incrementally, + instead of at the end of development as autoscaling prepares for + release. This commit registers the autoscaling feature flag in REST + tests for non-snapshot builds. + +commit 90eb6a020dac8955c1cd99ee8f3c75d1f58bd9e4 +Author: Armin Braun +Date: Sun Feb 9 21:44:18 2020 +0100 + + Remove Redundant Loading of RepositoryData during Restore (#51977) (#52108) + + We can just put the `IndexId` instead of just the index name into the recovery soruce and + save one load of `RepositoryData` on each shard restore that way. + +commit 3e7f939f633121a210e642de19f8e63b2f8cbc48 +Author: Marios Trivyzas +Date: Sun Feb 9 18:00:36 2020 +0100 + + SQL: [Tests] Add more tests for aggs and literals (#52086) + + Add some more tests where more than one literal is selected, + unaliased and aliased. + + Follows: #42121 + (cherry picked from commit 405271d408a233e697eb2e9ded3005a71f4df5e7) + +commit c4c0db6f214bc93bd592632fea02f44f6975c87f +Author: Jason Tedor +Date: Sat Feb 8 18:50:14 2020 -0500 + + Introduce jvm.options.d for customizing JVM options (#51882) + + This commit introduces the ability to override JVM options by adding + custom JVM options files to a jvm.options.d directory. This simplifies + administration of Elasticsearch by not requiring administrators to keep + the root jvm.options file in sync with changes that we make to the root + jvm.options file. Instead, they are not expected to modify this file but + instead supply their own in jvm.options.d. In Docker installations, this + means they can bind mount this directory in. In future versions of + Elasticsearch, we can consider removing the root jvm.options file + (instead, providing all options there as system JVM options). + +commit 214beed90ff0470cf4cc92764a870aba0c4d4686 +Author: Costin Leau +Date: Sat Feb 8 23:10:08 2020 +0200 + + QL: move query AST from SQL to QL (#52069) + + (cherry picked from commit 59368968b698652352be1bb2a60d5a357a01b978) + +commit 9f541d909d852c806c20889677f479706c298ad3 +Author: Nhat Nguyen +Date: Sat Feb 8 11:26:18 2020 -0500 + + Always create search context for scroll queries (#52078) + + We need to either exclude null responses from the scroll search response + or always create a search context for every target shards, although that + scroll query can be written to match_no_docs. Otherwise, we won't find + search_context for subsequent scroll requests. + + This commit implements the latter option as it's less error-prone. + + Relates #51708 + +commit 7c2fae107040f39a487436709cecd8dfb46d6674 +Author: Jason Tedor +Date: Sat Feb 8 12:17:15 2020 -0500 + + Remove unused anyOf import in PackagingTestCase + + An import was left behind after resolving a merge conflict. This commit + removes that leftover import statement. + +commit 6d4cbe5dedee1c0c38c2dc9ce9f002e0fe10bb3c +Author: Jason Tedor +Date: Sat Feb 8 12:07:38 2020 -0500 + + Replace assertFalse/assertTrue for file existence (#52092) + + In the packaging tests, we added convenience methods for asserting file + existence and file non-existence. This commit replaces the remaining + uses of assertFalse/assertTrue in favor of these dedicated matchers. + +commit b77ef1f61bc945477a896a8540e06ea245ca7422 +Author: Armin Braun +Date: Sat Feb 8 12:14:51 2020 +0100 + + Cleanup some Dead Code in o.e.index.store (#52045) (#52084) + + One obviously unused method and an incorrect Javadoc that + referenced an otherwise unused class. + +commit e541ca794f6545f69b7819ab241a200cdca0edee +Author: Jason Tedor +Date: Fri Feb 7 21:42:31 2020 -0500 + + Fix compilation in FileExtensionMatchers + + Java 8 can not infer types as well as Java 11 does. This means a + backport of some code that relied on Java 11's superior abilities to + infer types caused Java 8's head to explode trying to infer the same + type in 7.x. This commit addresses this by giving Java 8 the types that + it needs. + +commit 8b1d2c5b950c41e395f3e15270c00c12c9746e38 +Author: Jason Tedor +Date: Fri Feb 7 21:35:43 2020 -0500 + + Permit autoscaling feature flag in release builds (#52088) + + This commit provides a path to set register the autoscaling feature flag + in release builds, and therefore enabling autoscaling in release + builds. The primary reason that we add this is so that our release docs + tests can pass. Our release docs tests do not have infrastructure in + place to only register snippets from included portions of the docs, they + instead include all docs snippets. Since autoscaling can not be enabled + in release builds, this meant that the autoscaling snippets would fail + in the release docs tests. To address then, we need the ability to + enable autoscaling in the release docs tests which we can now do with + the system property added here. This system property will be removed + when autoscaling is ready for release. + +commit ff56e44f744745234fca1c516f7c611cf852e7cb +Author: Jason Tedor +Date: Fri Feb 7 21:34:20 2020 -0500 + + Add file existence matchers in packaging tests (#52090) + + The packaging tests like to assert that files exist or do not exist. We + do this with assertFalse and assertTrue which leads to useless assertion + messages, especially when asserting a file does not exist, but it does + and it is a directory. This commit helps with this situation by adding + dedicated matchers. + +commit c75050b7fc38841a8dcad4c6466299f8a181b121 +Author: Jason Tedor +Date: Fri Feb 7 18:55:06 2020 -0500 + + Remove extraneous whitespace in docs/build.gradle + + There is some extraneous whitespace here, and every time I look at this + file my editor wants to make these changes and so my diffs end up having + this noise in it which I fight to exclude. This commit addresses this + issue by removing this extraneous whitespace. + +commit dffcd021dfd27058371b9375a1655a6079f438cd +Author: Benjamin Trent +Date: Fri Feb 7 17:17:11 2020 -0500 + + [7.x] [ML] Add bwc serialization unit test scaffold (#51889) (#52061) + + * [ML] Add bwc serialization unit test scaffold (#51889) + + Adds new `AbstractBWCSerializationTestCase` which provides easy scaffolding for BWC serialization unit tests. + + These are no replacement for true BWC tests (which execute actual old code). These tests do provide some good coverage for the current code when serializing to/from old versions. + + * removing unnecessary override for 7.series branch + + * adding necessary import + + Co-authored-by: Elastic Machine + +commit c6111eb90e98f658950bb72dacee1f7c03e02063 +Author: Benjamin Trent +Date: Fri Feb 7 17:04:58 2020 -0500 + + [ML][Inference] adding number_samples to TreeNode (#51937) (#52060) + + in preparation for feature importance and split information gain, adding `number_samples` field to `TreeNode` definition. + + Co-authored-by: Elastic Machine + +commit c8f0fe135d4530048de16e119b0ac08874e67c84 +Author: Zachary Tong +Date: Fri Feb 7 16:40:54 2020 -0500 + + Update release notes for BC5 + +commit e5a9e44ca4de423b4b9e0e2458b1b4346b3b7816 +Author: Mark Vieira +Date: Fri Feb 7 13:14:02 2020 -0800 + + Mute IndicesRequestCacheIT.testQueryRewriteDatesWithNow() + + Signed-off-by: Mark Vieira + +commit 8cf47aca7e885902190f9f5428df39ac4ea3a987 +Author: Przemko Robakowski +Date: Fri Feb 7 21:13:25 2020 +0100 + + [7.x] Improve Painless compilation performance for nested conditionals (#52056) (#52074) + + * Improve Painless compilation performance for nested conditionals (#52056) + + This PR changes how conditional expression is handled in `PainlessParser` + in a way that avoids the need for backtracking, which led to exponential + compilation times in case of nested conditionals. + + The test was added ensures that we can compile deeply nested conditionals. + + Co-authored-by: Elastic Machine + + * Fix Map.of in Java8 + + Co-authored-by: Elastic Machine + +commit c827f6f440e4c50649ed710e205498ac3e1a91fc +Author: Przemko Robakowski +Date: Fri Feb 7 21:00:39 2020 +0100 + + Avoid clash between source field and header field in CsvProcessorTests (#51962) (#52070) + + This change fixes flakiness in `CsvProcessorTests` where source field + can be the same as one of the headers used by tests which messes up + asserts when we check that field is not present after processor run. + + Closes #50209 + +commit 2588022b81ae28a9d2cf9bca9994081a338622be +Author: debadair +Date: Fri Feb 7 11:03:56 2020 -0800 + + [DOCS] Fixed typo. (#52071) + +commit 337d73a7c6f8c5684da1cd7c69a2c236b21897fc +Author: Julie Tibshirani +Date: Fri Feb 7 10:24:01 2020 -0800 + + Rename MapperService#fullName to fieldType. + + The new name more accurately describes what the method returns. + +commit bc3ef9b8dfdf28d780990ca43779eeb233ca10f9 +Author: Alpar Torok +Date: Mon Oct 7 15:33:25 2019 +0300 + + Fix the windows packaging tests exit code + + $? is a boolean and was used incorrectly + +commit 282e9196073bf37e71ce1f6822449f0edb0fd152 +Author: Emanuele Sabellico +Date: Fri Feb 7 18:26:12 2020 +0100 + + SQL: [Tests] Add integ tests for selecting a literal and an aggregate (#42121) + + The related issue regarding aggregation queries where some literals + are also selected together with aggregate function has been fixed + with #49570. Add integration tests to verify the behavior. + + Relates to: #41411 + + (cherry picked from commit 9f414a8d05c75e1a9f8250084f6dcd634d5d78d8) + +commit 7c6264b28c5ac104117f296f4e7aa811dfd3ee09 +Author: Tanguy Leroux +Date: Thu Feb 6 17:38:44 2020 +0100 + + Mute IndicesRequestCacheIT.testQueryRewrite() + + Relates #32827 + +commit db22fb6e1c7b9aba8efd19a42fa74c7cbf643f12 +Author: James Rodewig +Date: Fri Feb 7 12:14:44 2020 -0500 + + Revert "[DOCS] Include docs on permanently unreleased branches only (#51743)" + + This reverts commit 9d097968151bcf6d89357b8057cd00805a09061b. + +commit e4936230a319ecc19f0e75e0a08da95ca5c6f7ce +Author: Raidok +Date: Fri Feb 7 16:40:16 2020 +0200 + + [DOCS] Fix "Asynchronous usage" title in HLRC docs (#52017) + +commit 4add82d9662a0e28426b5417956d3f08a9e676c8 +Author: Albert Zaharovits +Date: Fri Feb 7 13:35:26 2020 +0200 + + Mute CoreFullClusterRestartIT testRecovery (#52038) + + Relates #52031 + +commit 8f10a7c6ca883ca06318ff85aea98012e5ad30b3 +Author: David Kyle +Date: Fri Feb 7 10:07:18 2020 +0000 + + [ML] Make Ensemble feature names optional (#51996) + + The featureNames field is requisite in individual models but is not required by the Ensemble. + +commit 91e938ead8199a05ea9e99dda839245a5305e139 +Author: Armin Braun +Date: Fri Feb 7 09:03:20 2020 +0100 + + Add Trace Logging of REST Requests (#51684) (#52015) + + Being able to trace log all REST requests to a node would make debugging + a number of issues a lot easier. + +commit 25daf5f1e10c05e5b960073d10f27e7534beaed9 +Author: Jason Tedor +Date: Thu Feb 6 21:55:01 2020 -0500 + + Add autoscaling API skelton (#51564) + + The main purpose of this commit is to add a single autoscaling REST + endpoint skeleton, for the purpose of starting to build out the build + and testing infrastructure that will surround it. For example, rather + than commiting a fully-functioning autoscaling API, we introduce here + the skeleton so that we can start wiring up the build and testing + infrastructure, establish security roles/permissions, an so on. This + way, in a forthcoming PR that introduces actual functionality, that PR + will be smaller and have less distractions around that sort of + infrastructure. + +commit 488944f4a1ddf2be89f8098218578f6112d4d3fe +Author: Andrei Stefan +Date: Fri Feb 7 04:15:59 2020 +0200 + + SQL: Handle uberjar scenario where the ES jdbc driver file is bundled in another jar (#51856) (#52024) + + (cherry picked from commit 6247b0793c9db19a8a9fa6f0164cc14d0debed6e) + +commit 16afbf91bb8aeb22933d21fe1012e2332468977c +Author: Mark Vieira +Date: Thu Feb 6 16:29:04 2020 -0800 + + Mute ReindexFailureTests test + +commit 9d097968151bcf6d89357b8057cd00805a09061b +Author: James Rodewig +Date: Thu Feb 6 14:44:20 2020 -0500 + + [DOCS] Include docs on permanently unreleased branches only (#51743) + + Adds the ability to display docs on permanently unreleased branches, + such as `master` and `7.x`. + + Also updates how the autoscaling and EQL docs are included. + Currently, these feature-flag docs would display on any unreleased + branches that contain the changes, such as 7.7. + +commit bc7aff917ee675c69c068672af8db021d954e476 +Author: Mark Vieira +Date: Thu Feb 6 11:04:16 2020 -0800 + + Mute LangPainlessClientYamlTestSuiteIT context API tests (#51939) (#52011) + +commit 2e444e77df7ddff06425a63d2ba81fe73fc435e1 +Author: Mark Vieira +Date: Fri Jan 24 16:33:31 2020 -0800 + + Avoid running docker-compose tasks concurrently in CI (#51439) + + + (cherry picked from commit a621b98f2f220c67843e87a75892738e98501330) + +commit b14b803676a6f39daadbc6c95bac1c8dec9bd42a +Author: Mark Vieira +Date: Thu Feb 6 10:47:14 2020 -0800 + + Mute failing tests on Windows (#50825) (#52010) + +commit 846f87a26e8bc750c84aa07a5886b45981329040 +Author: Benjamin Trent +Date: Thu Feb 6 12:10:18 2020 -0500 + + [ML] allow close/stop for jobs/datafeeds with missing configs (#51888) (#51997) + + If the configs are removed (by some horrific means), we should still allow tasks to be cleaned up easily. + + Datafeeds and jobs with missing configs are now visible in their respective _stats calls and can be stopped/closed. + +commit 03fb5cdaae30afb56c33a9c7e5c51d4c54db0c27 +Author: Hendrik Muhs +Date: Thu Feb 6 17:26:43 2020 +0100 + + fallback to float if source type is scaled_float for mapping deduction (#51990) + + fallback to float if source type is scaled_float for mapping deduction of min/max aggregation + + fixes #51780 + +commit 884d4904d462905f51891deb8513e9eeb83929bd +Author: Martijn Laarman +Date: Thu Feb 6 17:18:22 2020 +0100 + + Time parameter includes description (#49368) + + * Time parameter includes description + + In option enumeration causing codegenerators to pick up the description + as a value to send. + + * cat.shards missing ending quotes + + (cherry picked from commit 1c3b341960e3b70555927bdbab325d26382f68b2) + +commit f62670075798fba5e221e10b7a0df840f1b3628b +Author: Martijn Laarman +Date: Thu Feb 6 17:15:46 2020 +0100 + + Introduces validation to restrict API's to single namespace (#51982) + + To be merged after #51981 lands + + (cherry picked from commit 196eaf8ac1ccc777f5bd81469d1148bc7ec299dd) + +commit 898dd0b9ccfab42c8ece1a58e119b8d754e18e91 +Author: Martijn Laarman +Date: Thu Feb 6 13:15:56 2020 +0100 + + Cat.ml.* introduces an additional depths to namespace API's (#51981) + + Not all clients support this e.g if the java high level rest client were + to map this it would look like `client.cat().ml().api()` which hinders + discoverability. + + (cherry picked from commit 21cdabf09dc8305ce2f5e3b6cb193f67137d8bdb) + +commit 5092d3098d7a93e9c37489346f1105922264ee6c +Author: Ioannis Kakavas +Date: Thu Feb 6 13:05:09 2020 +0200 + + Cleanup test user in HLRC test (#49477) (#51942) + + SecurityIT.testGetUser creates a user for testing purposes, but did + not delete the user at the end of the test. This could leave the + cluster in an unexpected state for other tests. + + This commit: + - Deletes the user at the end of `testGetUser` + - Adds the test-name as metadata to the users that are created in `SecurityIT` + so that their origin is clear if they do interfere with other tests + - Enables SecurityDocumentationIT.testGetUsers on the expectation that + the new cleanup step will resolve the unreliability of that test. + + Relates: #48440 + + Co-authored-by: Tim Vernum + +commit cdb986249587682db77f58ac1593cfa41d17912e +Author: Rory Hunter +Date: Thu Feb 6 10:00:14 2020 +0000 + + Clarify use of ES_JAVA_OPTS and Docker (#51984) + + Backport of #51867. + + Tweak the documentation around configuring the heap size when using + Docker, to state that: + + - using `ES_JAVA_OPTS` is the preferred method + - Any `ES_JAVA_OPTS` overrides the defaults in `jvm.options` + - It's possible to bind-mount a custom `jvm.options` + +commit 0f333c89b9e6a1e10730fbda99334f355947f23f +Author: Jim Ferenczi +Date: Thu Feb 6 10:53:11 2020 +0100 + + Always rewrite search shard request outside of the search thread pool (#51708) (#51979) + + This change ensures that the rewrite of the shard request is executed in the network thread or in the refresh listener when waiting for an active shard. This allows queries that rewrite to match_no_docs to bypass the search thread pool entirely even if the can_match phase was skipped (pre_filter_shard_size > number of shards). Coordinating nodes don't have the ability to create empty responses so this change also ensures that at least one shard creates a full empty response while the other can return null ones. This is needed since creating true empty responses on shards require to create concrete aggregators which would be too costly to build on a network thread. We should move this functionality to aggregation builders in a follow up but that would be a much bigger change. + This change is also important for #49601 since we want to add the ability to use the result of other shards to rewrite the request of subsequent ones. For instance if the first M shards have their top N computed, the top worst document in the global queue can be pass to subsequent shards that can then rewrite to match_no_docs if they can guarantee that they don't have any document better than the provided one. + +commit fb710cc62bee6331e63f9bccd912f6dc929c647b +Author: Jim Ferenczi +Date: Thu Feb 6 08:25:29 2020 +0100 + + Remove the query builder serialization from QueryShardException message (#51885) + + QueryBuilders that throw exceptions on shards when building the Lucene query + returns the full serialization of the query builder in the exception message. + For large queries that fails to execute due to the max boolean clause, this means + that we keep a reference of these big messages for every shard that participate + in the request. In order to limit the memory needed to hold these query shard + exceptions in the coordinating node, this change removes the query builder + serialization from the shard exception. The query is known by the user so + there should be no need to repeat it on every shard exception. We could also + omit the entire stack trace for known bad request exception but it would deserve + a separate issue/pr. + + Closes #51843 + Closes #48910 + +commit 4c2dcf2bde81a3c1f9e29c484d9447741c851481 +Author: Lisa Cawley +Date: Wed Feb 5 19:00:28 2020 -0800 + + [DOCS] Adds curl explanation to getting started content (#51963) + +commit 53bd88ea8c6107c91a314c401dba1bd8ef99e393 +Author: Lisa Cawley +Date: Wed Feb 5 18:54:53 2020 -0800 + + [DOCS] Adds tip for elastic built-in user (#51891) + +commit 12473c2bcb297e02706a22f88e84274cb2febf6e +Author: Jason Tedor +Date: Wed Feb 5 20:46:54 2020 -0500 + + Log failure when cleaning shard follow task (#51971) + + When clenaing a shard follow task after an index has been deleted, an + exception can occur submitting the complete persistent task + action. However, this exception message is not logged. This commit + addresses this by including the exception that led to the failure in the + log message. + +commit 6332de40b4eabe4fc54ad729dff0af507bda4ec2 +Author: Przemko Robakowski +Date: Wed Feb 5 23:35:52 2020 +0100 + + Add empty_value parameter to CSV processor (#51567) (#51966) + + * Add empty_value parameter to CSV processor + + This change adds `empty_value` parameter to the CSV processor. + This value is used to fill empty fields. Fields will be skipped + if this parameter is ommited. This behavior is the same for both + quoted and unquoted fields. + + * docs updated + + * Fix compilation problem + + Co-authored-by: Elastic Machine + + Co-authored-by: Elastic Machine + +commit 69c1ecbf2bde032077a954e94b4c0bf2377fa4d1 +Author: Mark Vieira +Date: Wed Feb 5 14:00:13 2020 -0800 + + Add list of compatible OSes for vagrant packaging CI tests + +commit 80e29a47d88941c0397acbca89abab80a01e5fd3 +Author: Nik Everett +Date: Wed Feb 5 16:32:55 2020 -0500 + + Fix a sneaky bug in rare_terms (#51868) (#51959) + + When the `rare_terms` aggregation contained another aggregation it'd + break them. Most of the time. This happened because the process that it + uses to remove buckets that turn out not to be rare was incorrectly + merging results from multiple leaves. This'd cause array index out of + bounds issues. We didn't catch it in the test because the issue doesn't + happen on the very first bucket. And the tests generated data in such a + way that the first bucket always contained the rare terms. Randomizing + the order of the generated data fixed the test so it caught the issue. + + Closes #51020 + +commit ea1d9e080377811dddbf951a57e640b677182291 +Author: Lisa Cawley +Date: Wed Feb 5 12:23:30 2020 -0800 + + [DOCS] Defines data frame transform stats API objects (#44197) + +commit b93ebc29c1588429756b076b0985e221c6690d3d +Author: Lisa Cawley +Date: Wed Feb 5 11:07:02 2020 -0800 + + [DOCS] Augments update license API (#51903) + +commit f8c8a10f05847e6757f74bbaa3fbfed8b45a11b3 +Author: Lee Hinman +Date: Wed Feb 5 09:37:12 2020 -0700 + + Add documentation about ILM forcemerge with best_compression (#51893) + + This adds the option to the parameter list and a warning about the index being unavailable during + the close and open operations. + + Relates to #49974 + +commit d0865b963e3c74b26a8c0024051f6ecff6190210 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Feb 5 16:41:47 2020 +0100 + + Disable reindex test against 0.90 on mac (#51884) + + Follow-up to #51449 to also disable the test on mac. + + Closes #51617 + +commit d86a7ad6d2d8dca0b8e3ad3531d1588cb797cdf1 +Author: Tanguy Leroux +Date: Wed Feb 5 15:52:15 2020 +0100 + + Give more time to AutoFollowIT tests (#51938) + + AutoFollowIT tests are regularly failing on CI because they rely + on how cluster state updates are processed within the integration + clusters. We tried to limit this in #49141 by moving to latches + instead of waiting for assertions to pass but there are still some + places were it still need to wait for the cluster state updates to + be processed and auto-follow stats to be updated. + + This commit gives more time to assertBusy() that verifies the + AutoFollowStats (up to 60 seconds) and also always log the + auto-follow stats in case the assertions failed. + + Closes #48982 + +commit bd6d9e063c1158f11b1532249ec43d3f02bef5f9 +Author: Costin Leau +Date: Wed Feb 5 16:53:42 2020 +0200 + + EQL: Add missing commit messages for #51940 + + * EQL: Plug query params into the AstBuilder (#51886) + + As the eventType is customizable, plug that into the parser based on the + given request. + + (cherry picked from commit 5b4a3a3c07eacbc339cbd4c05a3621d056cc8d60) + + * EQL: Add field resolution and verification (#51872) + + Add basic field resolution inside the Analyzer and a basic Verifier to + check for any unresolved fields. + + (cherry picked from commit 7087358ae2fb212811d480ec8641a46167946c82) + + * EQL: Introduce basic execution pipeline (#51809) + + Add main classes that form the 'execution' pipeline are added - most of + them have no functionality; the purpose of this PR is to add flesh out + the contract between the various moving parts so that work can start on + them independently. + + (cherry picked from commit 9a1bae50a49af7fe8467b74b154c0d82c6bb9a19) + + * EQL: Add AstBuilder to convert to QL tree (#51558) + + * EQL: Add AstBuilder visitors + * EQL: Add tests for wildcards and sets + * EQL: Fix licensing + * EQL: Fix ExpressionTests.java license + * EQL: Cleanup imports + * EQL: PR feedback and remove LiteralBuilder + * EQL: Split off logical plan from expressions + * EQL: Remove stray import + * EQL: Add predicate handling for set checks + * EQL: Remove commented out dead code + * EQL: Remove wildcard test, wait until analyzer + + (cherry picked from commit a462700f9c8e1fb977d62d42eb0077403b8fa98b) + + * EQL grammar updates and tests (#49658) + + * EQL: Additional tests and grammar updates + * EQL: Add backtick escaped identifiers + * EQL: Adding keywords to language + * EQL: Add checks for unsupported syntax + * EQL: Testing updates and PR feedback + * EQL: Add string escapes + * EQL: Cleanup grammar for identifier + * EQL: Remove tabs from .eql tests + + (cherry picked from commit 6f1890bf2d52cabdfd1e7848fb481cf54b895f25) + +commit 6ff0e411a8d6a0186d4bcf8756cd208d694fb500 +Author: Costin Leau +Date: Wed Feb 5 16:45:58 2020 +0200 + + EQL: backport updates to 7.x (#51940) + +commit 3be70f64d88d10dd72d94fb34ccd5a8b788998c3 +Author: Armin Braun +Date: Wed Feb 5 15:37:06 2020 +0100 + + Fix GCS Mock Http Handler JDK Bug (#51933) (#51941) + + There is an open JDK bug that is causing an assertion in the JDK's + http server to trip if we don't drain the request body before sending response headers. + See https://bugs.openjdk.java.net/browse/JDK-8180754 + Working around this issue here by always draining the request at the beginning of the handler. + + Fixes #51446 + +commit 79f143907a6bbedbd52fc96ed2afd7084ffb54fa +Author: Benjamin Trent +Date: Wed Feb 5 08:26:44 2020 -0500 + + [7.x] [ML] add _cat/ml/trained_models API (#51529) (#51936) + + * [ML] add _cat/ml/trained_models API (#51529) + + This adds _cat/ml/trained_models. + +commit b70cbc97aaf32030c32b3cbf2fda86ef4ee02489 +Author: James Rodewig +Date: Wed Feb 5 08:12:09 2020 -0500 + + [DOCS] Add EQL syntax page (#51821) + + Adds documentation for basic EQL syntax. + + Joins, sequences, and other syntax to be added as its supported + in future development. + + Co-Authored-By: Ross Wolf <31489089+rw-access@users.noreply.github.com> + +commit 0610eb51efc8711cd9885f98fda513f6ffbb29ca +Author: Martijn van Groningen +Date: Wed Feb 5 13:27:31 2020 +0100 + + Change HLRC SourceExists to use GetSourceRequest instead of GetRequest (#51789) (#51913) + + Originates from #50885 + + Co-authored-by: Maxim + +commit 64f9a2089ba4470a574bb416ba641f09dbd63389 +Author: Marios Trivyzas +Date: Wed Feb 5 12:32:40 2020 +0100 + + SQL: [Tests] add tests for literals and GROUP BY (#51878) + + Add unit and integration tests where literals are SELECTed + in combination with GROUP BY and possibly aggregate functions. + + Relates to #41411 and #34583 + which have been fixed. + + (cherry picked from commit b97f1ca12675d6ea4772c60578922fe1cc2409ee) + +commit ababd730f61ca80339f98ca906c8dc22f803935d +Author: Ignacio Vera +Date: Wed Feb 5 12:35:49 2020 +0100 + + Histogram field: Use #name() instead of #simpleName() when generating doc values (#51920) (#51927) + +commit 289d4f4f4ddb3e67d66d8b119a38689a761289a8 +Author: David Kyle +Date: Wed Feb 5 10:49:36 2020 +0000 + + [ML] Remove stray field from inference docs (#51870) + + model_info_field is not a valid option + +commit 19ae3e6a48a1c3c2463b944c9c0d4cd715f3f063 +Author: Karel Minarik +Date: Wed Feb 5 11:29:21 2020 +0100 + + Fix the type for "slices" in the Reindex and Update By Query REST API specification (#51908) (#51911) + + This patch supplements #51792 and #51535 where the type of the "slices" parameter has been fixed. + + (cherry picked from commit 2ed9e95100474f3dfbeb7efb0529e237b8f61e53) + +commit 60c93b6df5016d414069b6a6053c9672eed3c989 +Author: Yannick Welsch +Date: Wed Feb 5 11:13:37 2020 +0100 + + Increase scroll timeout for upgrade test (#51912) + + Bumps the timeout already bumped in #50195, which was insufficient. + +commit ad9d2f1922d34fabadcaa5e465edd859f5e99178 +Author: Adrien Grand +Date: Wed Feb 5 11:02:25 2020 +0100 + + Move analysis/mappings stats to cluster-stats. (#51875) + + Closes #51138 + +commit 4def3694ab2cef36f70bfa5f3845743b3d7fe75f +Author: Andrei Dan +Date: Wed Feb 5 09:56:53 2020 +0000 + + Test: match the \ilm/ and \slm/ test name too (#51811) (#51919) + + We only drop ilm/slm policies on teardown only if the running docs tests + are ilm/slm related. + + This updates the test name pattern to match the ilm/slm related tests + when running on windows + (eg.`reference\ilm/update-lifecycle-policy/line_29`). + + (cherry picked from commit 4bb5bbd52eee59bd3eee6d766a9efc159822d9b9) + Signed-off-by: Andrei Dan + +commit b4480bb8a49d5bbe7d993e8ec2409df8106690ad +Author: Yannick Welsch +Date: Wed Feb 5 10:46:22 2020 +0100 + + Mute LoggingOutputStreamTests (#51917) + + Relates #51838 + +commit c0156cbb5d3c013d8e7309818a9db2d0ee22e0fc +Author: debadair +Date: Tue Feb 4 16:45:18 2020 -0800 + + Backporting updates to ILM org, overview, & GS (#51898) + + * [DOCS] Align with ILM API docs (#48705) + + * [DOCS] Reconciled with Snapshot/Restore reorg + + * [DOCS] Split off ILM overview to a separate topic. (#51287) + + * [DOCS} Split off overview to a separate topic. + + * [DOCS] Incorporated feedback from @jrodewig. + + * [DOCS] Edit ILM GS tutorial (#51513) + + * [DOCS] Edit ILM GS tutorial + + * [DOCS] Incorporated review feedback from @andreidan. + + * [DOCS] Removed test link & fixed anchor & title. + + * Update docs/reference/ilm/getting-started-ilm.asciidoc + + Co-Authored-By: James Rodewig + + * Fixed glossary merge error. + + Co-authored-by: James Rodewig + +commit 0be61a3662fbd0afc5d1af613c7a1f26adddaa8b +Author: Lee Hinman +Date: Tue Feb 4 14:15:43 2020 -0700 + + [7.x] Adding best_compression (#49974) (763480ee) (#51819) + + * Adding best_compression (#49974) + + This commit adds a `codec` parameter to the ILM `forcemerge` action. When setting the codec to `best_compression` ILM will close the index, then update the codec setting, re-open the index, and finally perform a force merge. + + * Fix ForceMergeAction toSteps construction (#51825) + + There was a duplicate force merge step and the test continued to fail. This commit clarifies the + `toStep` method and changes the `assertBestCompression` method for better readability. + + Resolves #51822 + + * Update version constants + + Co-authored-by: Sivagurunathan Velayutham + +commit 38ce4288316c287eb108a50904687f0600cbcaed +Author: Julie Tibshirani +Date: Tue Feb 4 11:24:57 2020 -0800 + + Create a class to hold field capabilities for one index. (#51844) + + Currently, the same class `FieldCapabilities` is used both to represent the + capabilities for one index, and also the merged capabilities across indices. To + help clarify the logic, this PR proposes to create a separate class + `IndexFieldCapabilities` for the capabilities in one index. The refactor will + also help when adding `source_path` information in #49264, since the merged + source path field will have a different structure from the field for a single index. + + Individual changes: + * Add a new class IndexFieldCapabilities. + * Remove extra constructor from FieldCapabilities. + * Combine the add and merge methods in FieldCapabilities.Builder. + +commit 8d3e73b3a032f4ea07676517c8dd5c158b8957bc +Author: Maria Ralli +Date: Tue Feb 4 10:48:13 2020 +0200 + + Add host address to BindTransportException message (#51269) + + When bind fails, show the host address in addition to the port. This + helps debugging cases with wrong "network.host" values. + + Closes #48001 + +commit 337153b29fd57527d82dae203913b9888c942da9 +Author: feifeiiiiiiiiii +Date: Tue Feb 4 23:58:53 2020 +0800 + + Throw better exception on wrong `dynamic_templates` syntax (#51783) + + Currently, a mappings update request, where dynamic_mappings is an object + instead of an array, results in a http response with a 500 code. This PR checks + for this condition and throws a MapperParsingException like we do for other + malformed mapping cases. + + Closes #51486 + +commit b7aace44f3571a2228a78d26259195c2df26408e +Author: Hendrik Muhs +Date: Tue Feb 4 16:13:11 2020 +0100 + + mark transform API's stable (#51862) + + mark transform API's stable, meaning making transform GA for the next minor release + +commit 43a7aadd4699d2df736e92cd8906d28430ee91a9 +Author: Florian Kelbert +Date: Tue Feb 4 15:23:43 2020 +0100 + + [DOCS] Remove unneeded comma from CSV processor example (#51859) + +commit 28643f8df1f73a8e628af9a554545d1760509ef1 +Author: Dan Hermann +Date: Tue Feb 4 08:25:24 2020 -0600 + + Missing suffix for German Month "Juli" in Grok Pattern MONTH (#51579) (#51591) (#51863) + +commit 60a53f2897b5a2ff27cbca52b5eb5b7fd81120f8 +Author: baifan <285698756@qq.com> +Date: Tue Feb 4 22:15:06 2020 +0800 + + [DOCS] Fix `disk.used_percent` typo in `_cat/nodes` docs (#51854) + + Corrects an example for the `disk.used_percent` parameter in `_cat/nodes` API. + +commit 87b126bbfcd6a27ada366cbe9e71deb613092718 +Author: Grzegorz Banasiak +Date: Tue Feb 4 14:36:16 2020 +0100 + + [DOCS] Fix index_prefixes link in 'faster prefix queries' docs (#51833) + + Fixes a link in 'faster prefix queries' which incorrectly redirects to index_phrases mapping parameter description instead of index_prefixes. + +commit 9d55c45b5ab3cd834afc8da316e1f7eae0e2f42a +Author: David Roberts +Date: Tue Feb 4 12:36:36 2020 +0000 + + [ML] Improve multiline_start_pattern for CSV in find_file_structure (#51737) + + The work to switch file upload over to treating delimited files + like semi-structured text and using the ingest pipeline for CSV + parsing makes the multi-line start pattern used for delimited + files much more critical than it used to be. + + Previously it was always based on the time field, even if that + was towards the end of the columns, and no multi-line pattern + was created if no timestamp was detected. + + This change improves the multi-line start pattern by: + + 1. Never creating a multi-line pattern if the sample contained + only single line records. This improves the import + efficiency in a common case. + 2. Choosing the leftmost field that has a well-defined pattern, + whether that be the time field or a boolean/numeric field. + This reduces the risk of a field with newlines occurring + earlier, and also means the algorithm doesn't automatically + fail for data without a timestamp. + +commit c2b08bb72f094f99620bfafdd7310d255a1b2526 +Author: Hendrik Muhs +Date: Tue Feb 4 12:01:30 2020 +0100 + + [Transform] add support for percentile aggs (#51808) + + make transform ready for multi value aggregations and add support for percentile + + fixes #51663 + +commit 5d5f3ce2568d6f13cc49193602f55a8111a5337e +Author: Hendrik Muhs +Date: Tue Feb 4 10:35:31 2020 +0100 + + [Transform] improve irrecoverable error detection + + treat resource not found and illegal argument exceptions as irrecoverable error + + relates #50135 + +commit ba2810f23d78266b9608245a7aa42124fef7b962 +Author: William Brafford +Date: Mon Feb 3 18:07:26 2020 -0500 + + Use standard format for reload settings API (#51560) (#51828) + + * Use standard format for reload settings API + + The reload-secure-settings API page was not reorganized for the standard + API format, so this commit is reorganizing the page and adding some + links to the page in related documentation. + + * Fix broken links + + * Reorder examples to correctly check API response + + * Note that only certain settings are reloadable + + * [DOCS] Edits layout + + * [DOCS] Removes unnecessary callouts + + Co-authored-by: Lisa Cawley + + Co-authored-by: Lisa Cawley + +commit d293980a09d58dead747656864c61f0bd9d14972 +Author: Benjamin Trent +Date: Mon Feb 3 17:16:33 2020 -0500 + + [7.x] [ML] add GET _cat/ml/datafeeds (#51500) (#51829) + + * [ML] add GET _cat/ml/datafeeds (#51500) + + This adds GET _cat/ml/datafeeds && _cat/ml/datafeeds/{datafeed_id} + + * fixing for java8 compilation + +commit cff3fdb09105700cd6f955358a3dc7e0abfbf511 +Author: Igor Motov +Date: Mon Feb 3 09:47:58 2020 -0500 + + Fix version parser used by gradle (#51773) + + Due to of a typo in the version regex pattern only the last digit of a major + version number is taken into consideration. So docker's version 17.0.1 is parsed + as 7.0.1. + +commit 8fa4a40bdfa70ce94821507311d57a1c2151651a +Author: Jonathan Budzenski +Date: Mon Feb 3 10:52:57 2020 -0600 + + [rest spec] fill in documentation links for security.{put,delete}_privileges (#48482) + + Co-authored-by: Elastic Machine + +commit 8aaca4592245161dfba32c9c022ba03c45c132a5 +Author: Jochem Wichers Hoeth +Date: Mon Feb 3 16:34:34 2020 +0100 + + Fix section header in Get API key information API doc (#51807) + +commit 41552359a22f5530750f24a6fa08bab30ab47975 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Feb 3 15:48:19 2020 +0100 + + Increase master disruption test assert timeouts (#51810) + + After #51803, the timeouts waiting for assertions around master change + were too short. + +commit 4ea7297e1eedf839267c1935320351b3d8db0f5e +Author: James Rodewig +Date: Mon Feb 3 09:50:11 2020 -0500 + + [DOCS] Change http://elastic.co -> https (#48479) (#51812) + + Co-authored-by: Jonathan Budzenski + +commit 4083eae0b762dba063dd1ccc5b236da9582fda1a +Author: Dan Hermann +Date: Mon Feb 3 07:42:30 2020 -0600 + + [7.x] Secure password for monitoring HTTP exporter (#51775) + + Adds a secure and reloadable SECURE_AUTH_PASSWORD setting to allow keystore entries in the form "xpack.monitoring.exporters.*.auth.secure_password" to securely supply passwords for monitoring HTTP exporters. Also deprecates the insecure `AUTH_PASSWORD` setting. + +commit 1545c2ab26791628a3a79943ada09a1c750e6eb5 +Author: James Rodewig +Date: Mon Feb 3 08:33:02 2020 -0500 + + [DOCS] Document node stats response meta (#51263) + + Documents several metadata-related parameters returned by the + `GET _nodes/stats` API. + +commit 1800b2730fedadd9d84480418d32254df6afb992 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Feb 3 14:02:31 2020 +0100 + + Fix completeWith exception handling (#51734) + + ActionListener.completeWith would catch exceptions from + listener.onResponse and deliver them to lister.onFailure, essentially + double notifying the listener. Instead we now assert that listeners do + not throw when using ActionListener.completeWith. + + Relates #50886 + +commit a6d24d6a460dc2bdce34f56add704a27d51d7819 +Author: Przemyslaw Gomulka +Date: Mon Feb 3 14:17:43 2020 +0100 + + Fix ingest timezone logic backport(#51215) (#51802) + + when a timezone is not provided Ingest logic should consider a time to be in a timezone provided as a parameter. + When a timezone is provided Ingest should recalculate a time to the timezone provided as a parameter + + closes #51108 + backport(#51215) + +commit 918dfaff1fbfb654e6036965ce59768ec965a146 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Feb 3 13:57:18 2020 +0100 + + Increase disruption test publish timeout to 5s (#51803) + + With the new mechanism for storing cluster state in lucene, we store + index metadata in multiple data paths too. This causes cluster state + publish to timeout too frequently with a 1s timeout, so increasing it to + 5s. Also increasing follower check timeout to 5s since it also sometimes + has fsync in its timeout path and leader check for symmetry. + + Closes #51329 + +commit 81388051d8d39ddbdac87d0433616f59698dd2c9 +Author: Andrei Dan +Date: Mon Feb 3 11:54:00 2020 +0000 + + Reenable testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy (#51768) (#51801) + + We suspect the flakiness could’ve come from the fact that the rollover + step used to create the new index and roll the write alias to the new + index in separate cluster state updates. So the assertion that the + rolled index exists could’ve passed in the test but, before the + alias was rolled over to the new index, the subsequent write we execute + in the test (namely + `indexDocs("test_user", "x-pack-test-password", "foo_alias", 1)`) + would’ve sent the new document to the source index (ie. foo-logs-000001) + + This would see the source index containing 3 documents and the rolled + index (foo-logs-000002) 0 documents. + + However, we fixed this and the rollover step executes the “create index + and roll alias” in one single cluster update, so this situation should + not occur anymore. + + (cherry picked from commit 834261c4fe7dd93f437eeec43c00d01ff2279f86) + Signed-off-by: Andrei Dan + +commit d5d8fb26fa0f239cc551abfb34d61704e3f0be9a +Author: David Roberts +Date: Mon Feb 3 11:00:25 2020 +0000 + + [TEST] Remove obsolete test trace logging from NetworkDisruptionIT (#51746) + + The issue this logging was added to fix (#49908) was closed in + December and the problem has not recurred so this logging is no + longer needed. + +commit 39b471008192f4904aa135c13ed956e6a92f691e +Author: Yannick Welsch +Date: Mon Feb 3 09:44:55 2020 +0100 + + Add Docker-based tests for proxy connection mode (#51658) + + Adds more tests for the new "proxy" remote cluster connection mode, using a Docker-based + setup, as well as testing SNI-based routing using HAProxy. + + - Checks that the new proxy mode can work in situations where the publish host of the nodes in + the remote cluster are not routable. + - Checks that the new proxy mode can work with clusters where nodes are hidden behind + HAProxy. + - Checks that the new proxy mode can work with clusters where nodes are hidden behind + HAProxy, using SNI to identify the nodes/cluster behind HAProxy. + + Relates #49067 + +commit 493f77d07614776b3d900640052d7a701036490f +Author: Karel Minarik +Date: Sun Feb 2 19:30:14 2020 +0100 + + Fix the type for "slices" in the Delete By Query REST API specification (#51792) (#51793) + + The previous patch in c1d9966d35d incorrectly set the `type` to `number|auto`, + which is incorrect — the "polymorphic" type, denoted with the `|` sign, + should contain only other types, ie. number, string, bool, etc. + + Fixes #51535 + + (cherry picked from commit 68db7fc611622ca0e418f454249e376e01f80587) + +commit 050c4d4c890699c4b84790cc5d6b78a5581843ee +Author: Karel Minarik +Date: Sun Feb 2 15:28:08 2020 +0100 + + Fixes for the REST specification (#51791) + + * REST: Test: Fix the `accept_enterprise` parameter for Get License API (#51527) + + The Get License API specifies the `accept_enterprise` parameter as a `boolean`: + + https://github.com/elastic/elasticsearch/blob/0ca5cb8cb636a4be9c36b8e38e565af66abc423b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get.json#L22-L27 + + In the test, a `string` is passed however, which makes the test compilation fail in the Go client. + + (cherry picked from commit e2a2169b3d44592057c143253bb56375ed3e4268) + + * Fix the SQL API documentation in REST specification (#51534) + + This patch fixes the SQL REST API documentation to conform to the current schema. + + (cherry picked from commit c8b6a849852699883086a6ada42279f2f68d7e07) + + * Fix the "slices" parameter for the Delete By Query API in the REST specification (#51535) + + This patch updates the `type` parameter in the Delete By Query API: according to + [the documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html#docs-delete-by-query-slice), + it can be set to "auto", but the type in the documentation allows only numerical values. + + This prevents people from setting the parameter to "auto" eg. in the Go client, + which generates source from the specification, and sets the corresponding Go + type as number. + + The patch uses the `|` notation, which we have discussed previously for encoding + a "polymorphic" parameter like this. + + Related: https://github.com/elastic/go-elasticsearch/issues/77 + + * Fix the Enrich API documentation in REST specification (#51528) + + This patch fixes the REST API documentation for the Enrich APIs to conform to the current schema. + + (cherry picked from commit 59f28f4f2feeba3f6d2f0b632410577eacb28121) + +commit ed170cc548755503d3ad845bde83f80ef202a3fb +Author: Hendrik Muhs +Date: Sat Feb 1 19:34:58 2020 +0100 + + [Transform] Fix stats can return old state information if security is enabled (#51732) (#51738) + + do index refresh of the internal transform index with the system user + instead of using the calling user which does not have sufficient rights + if security is enabled + + fixes #51728 + +commit 21224caeaf2182a4f09782b53fff0008309da8c8 +Author: Ryan Ernst +Date: Fri Jan 31 16:34:27 2020 -0800 + + Remove comparison to true for booleans (#51723) + + While we use `== false` as a more visible form of boolean negation + (instead of `!`), the true case is implied and the true value does not + need to explicitly checked. This commit converts cases that have slipped + into the code checking for `== true`. + +commit 61622c4f0cbebf45769406d13fcc516438e0d755 +Author: Ryan Ernst +Date: Fri Jan 31 16:28:43 2020 -0800 + + Fix LoggingOutputStream to work on windows (#51779) + + LoggingOutputStream reads a stream and breaks on newlines. This commit + fixes the behavior to account for windows newlines also containing `\r`. + + closes #51532 + +commit 4594a210bff1c111c93d4137d761cfcab9437af6 +Author: Lee Hinman +Date: Fri Jan 31 15:38:28 2020 -0700 + + [7.x] Fix SnapshotLifecycleRestIT.testFullPolicySnapshot (#517… (#51778) + + * Fix SnapshotLifecycleRestIT.testFullPolicySnapshot + + This previously was missing some key information in the output of the failure. This captures that + information and adds logging at each step so we can determine the cause *if* it fails again. + + Resolves #50358 + +commit 2e0bba0f63736a7509027553e7d73b1301639cd1 +Author: Nik Everett +Date: Fri Jan 31 16:44:30 2020 -0500 + + Skip date_histogram on range fields before 7.4.1 (#51771) + + range fields didn't have doc_values before 7.4.1. They were added in + #47472. + +commit d4f6f381506c377643e9a0334f25f7ad382656b4 +Author: Aleksandr Maus +Date: Fri Jan 31 15:14:27 2020 -0500 + + EQL: Fix #51541: [CI] unknown setting [xpack.eql.enabled] in release-tests (#51699) (#51770) + + Fixes #51541 + Co-authored-by: Igor Motov + +commit 16ef6a5785e390ecdc3a2da5bdfddd4e33c93dc6 +Author: Mayya Sharipova +Date: Fri Jan 31 14:46:29 2020 -0500 + + Mute testEs090 and testEs090WithFunnyThrottle + + Mure tests that reproducilbly fail. + + Related issue #51617 + +commit 70bb7c862d4def194d1f452eea72516e155ad249 +Author: Noor +Date: Fri Jan 31 13:29:25 2020 -0500 + + Fixed typo in comment (#51745) + + Comment said the supporter highlighter type was fvj, there's no such highlighter. It is supposed to say fvh for fast vector highlighting. + +commit 480e9238a4431e5345a8f7ae42ea0044be657f13 +Author: Darren LaCasse +Date: Fri Jan 31 12:27:37 2020 -0600 + + [DOCS] Remove extra word (#51757) + +commit 55b5c8f7032b6fb56ea2e23ac3160b430449944c +Author: Dimitris Athanasiou +Date: Fri Jan 31 20:28:29 2020 +0200 + + [7.x][ML] Remove index.unassigned.node_left.delayed_timeout setting from M… (#51740) (#51764) + + This setting was introduced with the purpose of reducing the time took by + tests that shut nodes down. Tests like `MlDistributedFailureIT` and + `NetworkDisruptionIT`. However, it is unfortunate to have to set the value + to an explicit value in production. In addition, and most important, the dynamically + choosing the value for this setting makes it impossible to adopt static index template configs + that we register via `IndexTemplateRegistry`, which we need to use in order to start + registering ILM policies for the ML indices. + + This commit removes this setting from our templates. I run the tests a few times and could + not see execution time differing significantly. + + Backport of #51740 + +commit 5ca51562ec62ea8ddbaa53d1eaaedec32ed38cb9 +Author: Andrei Dan +Date: Fri Jan 31 18:18:00 2020 +0000 + + Fix testThatNonExistingTemplatesAreAddedImmediately (#51668) (#51752) + + This addresses another race condition that could yield this test flaky. + + (cherry picked from commit d20d90aceb2b687239654d6f013f61f7f4cc1512) + Signed-off-by: Andrei Dan + +commit 20f47b14b030b49543cda64242886495170c1222 +Author: Andrei Dan +Date: Fri Jan 31 18:17:38 2020 +0000 + + Fix SnapshotLifecycleServiceTests.testPolicyCRUD (#51653) (#51755) + + (cherry picked from commit 8f9a87fa576a8a1c6ea3efb29bf1296d50d89ace) + Signed-off-by: Andrei Dan + +commit 227621dd1327dad17ef3e251d85fe01b65962d64 +Author: Przemko Robakowski +Date: Fri Jan 31 18:56:50 2020 +0100 + + Change index.lifecycle.step.master_timeout to indices.lifecycle.step.master_timeout (#51744) (#51761) + + * Change index.lifecycle.step.master_timeout to indices.lifecycle.step.master_timeout + + This changes setting name from `index.lifecycle.step.master_timeout` to + `indices.lifecycle.step.master_timeout` to avoid confusion about its scope. + `index.*` settings are recognized as index level settings, this one is node level. + + Reletes to #51698 + +commit deefc85d6052b1f8bf116b98f04ed31296cbdcf6 +Author: Lee Hinman +Date: Fri Jan 31 10:36:41 2020 -0700 + + [7.x] Stop policy on last PhaseCompleteStep instead of Termina… (#51758) + + Currently when an ILM policy finishes its execution, the index moves into the `TerminalPolicyStep`, + denoted by a completed/completed/completed phase/action/step lifecycle execution state. + + This commit changes the behavior so that the index lifecycle execution state halts at the last + configured phase's `PhaseCompleteStep`, so for instance, if an index were configured with a policy + containing a `hot` and `cold` phase, the index would stop at the `cold/complete/complete` + `PhaseCompleteStep`. This allows an ILM user to update the policy to add any later phases and have + indices configured to use that policy pick up execution at the newly added "later" phase. For + example, if a `delete` phase were added to the policy specified about, the index would then move + from `cold/complete/complete` into the `delete` phase. + + Relates to #48431 + +commit 3147453600270636be36354bcec223ca09cc5b70 +Author: Zachary Tong +Date: Fri Jan 31 11:39:43 2020 -0500 + + Update Release Notes for BC4 + +commit 42b885f0504e411c59b60741a6e22e8db2bbf596 +Author: Mayya Sharipova +Date: Fri Jan 31 11:20:15 2020 -0500 + + Upgrade to lucene-8.5.0-snapshot-3333ce7da6d (#51749) + + Backport for #51327 + +commit 1a9e7b2d691ad1cc9ed11c5b62f5c289dddf011b +Author: William Brafford +Date: Fri Jan 31 10:24:04 2020 -0500 + + Use an options loop in Elasticsearch startup script (#51547) (#51621) + + * Use loop to parse options rather than grep + + * Add test for --help flag with encrypted keystore + +commit 39a3a950de06bc277dc8fd4ac434ef195b4ccf53 +Author: David Turner +Date: Fri Jan 31 14:34:45 2020 +0000 + + Simplify rebalancer's weight function (#51632) + + This commit inlines the `weightShardAdded` and `weightShardRemoved` methods + from the `BalancedShardsAllocator#WeightFunction` that respectively add and + subtract 1 (±ε) from the result of `weight`. It then follows up with a number + of simplifications that this inlining enables. + + As a side-effect it also somewhat reduces the number of calls to canRebalance + and canAllocate during rebalancing when there are multiple shards of the same + index on a node that is heavier than average. + +commit 86f3b47299b0f2be21b18df8b2be94c27f923fbd +Author: Christoph Büscher +Date: Fri Jan 31 15:35:05 2020 +0100 + + Make `date_range` query rounding consistent with `date` (#50237) (#51741) + + Currently the rounding used in range queries can behave differently for `date` + and `date_range` as explained in #50009. The behaviour on `date` fields is + the one we document in https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html#range-query-date-math-rounding. + This change adapts the rounding behaviour for RangeType.DATE so it uses the + same logic as the `date` for the `date_range` type. + + Backport of #50237 + +commit e372854d43b51a3aae05d8d192b2b4872aaa550b +Author: Benjamin Trent +Date: Fri Jan 31 07:52:19 2020 -0500 + + [ML][Inference] Fix model pagination with models as resources (#51573) (#51736) + + This adds logic to handle paging problems when the ID pattern + tags reference models stored as resources. + + Most of the complexity comes from the issue where a model stored as a resource could be at the start, or the end of a page or when we are on the last page. + +commit dfc9f2330cf6a4bebf98be7543db73b68d6d0f3b +Author: István Zoltán Szabó +Date: Fri Jan 31 13:12:24 2020 +0100 + + [DOCS] Adds PUT inference API docs (#51231) + + Co-authored-by: Benjamin Trent + Co-authored-by: Lisa Cawley + +commit 9600ab4f578d112d10f7b9a1bea1241f4ebdb07a +Author: István Zoltán Szabó +Date: Fri Jan 31 12:51:46 2020 +0100 + + [DOCS] Adds recommendation on dedicated master-eligible nodes (#51674) + + Co-Authored-By: James Rodewig + +commit d7e1215e429aff40157eb016e581bfadd026ba0f +Author: Dominic Page <11043991+djptek@users.noreply.github.com> +Date: Fri Jan 31 11:55:10 2020 +0100 + + Backport of #50737 to 7.x (#51662) + + * Refactor GeoShape tests to GeoShape and GeoPoint (#50737) + + Backport to 7.x + +commit 915a931e93d22b2dcf6110dc976f486edfddee59 +Author: Adrien Grand +Date: Fri Jan 31 11:30:51 2020 +0100 + + Bucket aggregation circuit breaker optimization. (#46751) (#51730) + + Co-authored-by: Howard + +commit dc99acee66ed25163a965f8f6c7b06c7d3565067 +Author: Sven Schliesing +Date: Fri Jan 31 10:32:40 2020 +0100 + + [Docs] Fix typo in node-tool.asciidoc (#51667) + +commit 1dc965f03f271330fd0f779f99d57aacb61f36e5 +Author: Ioannis Kakavas +Date: Fri Jan 31 09:35:20 2020 +0200 + + Mute ec2 test in FIPS 140 mode (#51686) (#51726) + + as it needs an extra permission, until we can figure out how to + grant the permission in FIPS 140 mode too. + + See: https://github.com/elastic/elasticsearch/issues/51685 + +commit 282ae8fd8c7da9168591afb87249df954b78c0ce +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 30 16:32:42 2020 +0100 + + Increase log level for failing AbstractDisruptionIT tests (#51462) + + Increase log level for two failing tests to include trace logging for + PersistedClusterStateService. + + Relates #51329 + +commit 46ffc57abe0ba79df322e0e904b9d01068f853ba +Author: Ioannis Kakavas +Date: Fri Jan 31 09:23:02 2020 +0200 + + Fix test compilation error + +commit 72ae0ca73feccb969365fc0218df24a09cb13c24 +Author: David Turner +Date: Fri Jan 31 01:12:05 2020 +0000 + + Log exceptions in TcpTransport at DEBUG level (#51612) + + When running Elasticsearch on a flaky network, we may see nodes leaving the + cluster with reason `disconnected`. It may be useful to the cluster + administrator to see the full exception that caused the disconnection, but this + is only available with `TRACE` level logging which commingles the details of + the problem with other messages that are not useful to end users. + + This commit promotes logging of exceptions in `TcpTransport` from `TRACE` to + `DEBUG` to separate them from the truly `TRACE`-level messages. + +commit 77b00fc0c0390260e23c55586ca372c93836cbf3 +Author: Yang Wang +Date: Fri Jan 31 12:32:37 2020 +1100 + + Add warnings for invalid realm order config (#51195) (#51515) + + The changes are to help users prepare for migration to next major + release (v8.0.0) regarding to the break change of realm order config. + + Warnings are added for when: + * A realm does not have an order config + * Multiple realms have the same order config + + The warning messages are added to both deprecation API and loggings. + The main reasons for doing this are: 1) there is currently no automatic relay + between the two; 2) deprecation API is under basic and we need logging + for OSS. + +commit 11e86b13b4d316ad8a8b8d240e58728686e7b8e5 +Author: Mark Vieira +Date: Thu Jan 30 17:23:23 2020 -0800 + + Disable build cache for packaging tests (#51717) + +commit 6c879ac63cbeed48020484a66774a9e77fdfe9b0 +Author: Mark Vieira +Date: Thu Jan 30 17:06:56 2020 -0800 + + Test cluster throttle should never be less than 1 (#51713) + +commit 6b18cab826efa6388cb28ea601db8bcb32f58442 +Author: Mark Vieira +Date: Tue Jan 28 14:46:53 2020 -0800 + + Avoid NPE in Gradle build when max-workers is set to 1 (#51561) + +commit 1a40ebfa6707c5d74571dc0e77fa16d7d7106809 +Author: Lisa Cawley +Date: Thu Jan 30 16:13:26 2020 -0800 + + [DOCS] Adds missing testenv attribute (#51719) + +commit 10c81793512c7e38048237e8deb19a5c9880a2f1 +Author: Gordon Brown +Date: Thu Jan 30 16:31:27 2020 -0700 + + Use exclusions list instead of fake system indices (#51586) + + This commit switches the strategy for managing dot-prefixed indices that + should be hidden indices from using "fake" system indices to an explicit + exclusions list that must be updated when those indices are converted to + hidden indices. + +commit 8d2370bf00874fbe7804cbfeb9bba618bde5673c +Author: Mark Vieira +Date: Thu Jan 30 14:35:43 2020 -0800 + + Always use bundled JDK for external cluster nodes when BWC testing (#51505) (#51701) + +commit f1173aaa486a32c5145f10db2bdd131103c4c639 +Author: Bogdan Pintea +Date: Thu Jan 30 23:29:39 2020 +0100 + + SQL: Add optimisations for not-equalities (#51088) (#51700) + + * Optimize not-equalities in con-/disjunctions + + This commit adds optimisations of not-equalities in conjunctions and + disjunctions: + * for conjunctions, the not-equality can be optimized away when applied + together with a range or inequality, in case the not-equality point + falls outside the domain of the later condition; if its on the boarder, + it will modify the bound, to simply exclude the equality, if present; + otherwise no optimisation can be applied; + * for disjunctions, the not-equals could filter away the ranges and + inequalities, unless these include an equality on the bound, in which + case the entire condition becomes always true, but this would influence + the score() function, so it's been omitted; + + * fix aggregations of inequalities in ranges + + This commit fixes the loop that aggregates inequalities into ranges: + - it won't advance the outer loop index in case of a merge, since the + current element is removed; + - it will break the inner loop, since comparision against the element + selected in the outer loop can't continue, as it had been removed. + + + + (cherry picked from commit 789724ac2cc726de603849b4eeb8194da7528bcc) + +commit b9faa0733d9e54977d28613f58137dde44093f79 +Author: Lee Hinman +Date: Thu Jan 30 15:27:44 2020 -0700 + + [7.x] Rename ILM history index enablement setting (#51698) (#51705) + + * Rename ILM history index enablement setting + + The previous setting was `index.lifecycle.history_index_enabled`, this commit changes it to + `indices.lifecycle.history_index_enabled` to indicate this is not an index-level setting (it's node + level). + +commit 1380dd439ab85abb818c8a82ba57384902472526 +Author: Benjamin Trent +Date: Thu Jan 30 15:33:25 2020 -0500 + + [7.x] [ML][Inference] Fix weighted mode definition (#51648) (#51695) + + * [ML][Inference] Fix weighted mode definition (#51648) + + Weighted mode inaccurately assumed that the "max value" of the input values would be the maximum class value. This does not make sense. + + Weighted Mode should know how many classes there are. Hence the new parameter `num_classes`. This indicates what the maximum class value to be expected. + +commit 69ef9b05cd89be3ad12bce09e911c5a7c85370e4 +Author: Nhat Nguyen +Date: Thu Jan 30 09:02:38 2020 -0500 + + Increase shard inactive time to 1h in upgrade tests (#51651) + + testRecovery relies on the fact that shards are not flushed on inactive. + Our CI recently was too slow. It took more than 20 minutes to complete + the full cluster restart suite. This slowness caused some shards of + testRecovery were flushed on inactive. + + This commit increases the inactive time to 1h to reduce this noise. + + Closes #51640 + +commit 1cba5d7c4bd1b0e20c1d5170a454629f313b0589 +Author: Nhat Nguyen +Date: Wed Jan 29 16:27:33 2020 -0500 + + Force flush in FrozenEngine#testSearchers (#51635) + + We need to force flush to make the last commit safe; otherwise, we might + fail to open FrozenEngine. Note that we force flush before closing a + shard. + + Closes #51620 + +commit 72c5bc9630040a3048bbb5b8f6a85126a30d0da1 +Author: Ioannis Kakavas +Date: Thu Jan 30 21:35:03 2020 +0200 + + Disable diagnose trust mananer only when needed (#51683) + + Do not try to set xpack.ssl.security.diagnose.trust + when it is not registered + +commit bb963b996c942f62799a3dfe65d71979ba697bca +Author: Mark Vieira +Date: Thu Jan 30 09:34:55 2020 -0800 + + Support downloading JDKs with legacy version format (#51587) + + (cherry picked from commit b70f925ccb735dc84d59598de06df6bf35bd4bdc) + +commit 36b2663e98f80a324b1d268b9ac9e67d4d65dd3a +Author: James Rodewig +Date: Thu Jan 30 11:22:30 2020 -0500 + + [DOCS] Add attribute for Lucene analysis links (#51687) + + Adds a `lucene-analysis-docs` attribute for the Lucene `/analysis/` + javadocs directory. This should prevent typos and keep the docs DRY. + +commit 2a2a0941afda110cf5b8c41f9330ec3dc1786713 +Author: Benjamin Trent +Date: Thu Jan 30 10:16:14 2020 -0500 + + [ML][Inference] stream inflate to parser + throw when byte limit is reached (#51644) (#51679) + + Three fixes for when the `compressed_definition` is utilized on PUT + + * Update the inflate byte limit to be the minimum of 10% the max heap, or 1GB (what it was previously) + * Stream data directly to the JSON parser, so if it is invalid, we don't have to inflate the whole stream to find out + * Throw when the maximum bytes are reach indicating that is why the request was rejected + +commit 9c7a63214cef04d1f9ecde67056f50658fa76c80 +Author: Armin Braun +Date: Thu Jan 30 15:42:48 2020 +0100 + + Fix InternalEngineTests.testSeqNoAndCheckpoints (#51630) (#51671) + + * Fix InternalEngineTests.testSeqNoAndCheckpoints + + If we force flush while possibly triggering a merge the local checkpoint may change + from the expectation from the loop that just increments on every operation. + + Closes #51604 + +commit 4fcf5a9de4a0d3a40465975c6bf4ef48bd06a5ed +Author: James Rodewig +Date: Thu Jan 30 09:19:53 2020 -0500 + + [DOCS] Rewrite analysis intro (#51184) + + * [DOCS] Rewrite analysis intro. Move index/search analysis content. + + * Rewrites 'Text analysis' page intro as high-level definition. + Adds guidance on when users should configure text analysis + * Rewrites and splits index/search analysis content: + * Conceptual content -> 'Index and search analysis' under 'Concepts' + * Task-based content -> 'Specify an analyzer' under 'Configure...' + * Adds detailed examples for when to use the same index/search analyzer + and when not. + * Adds new example snippets for specifying search analyzers + + * clarifications + + * Add toc. Decrement headings. + + * Reword 'When to configure' section + + * Remove sentence from tip + +commit f373020349a7d3ab2a7fefa7ebe07f69b64d5f5a +Author: Marios Trivyzas +Date: Thu Jan 30 14:48:34 2020 +0100 + + SQL: Fix ORDER BY YEAR() function (#51562) + + Previously, if YEAR() was used as and ORDER BY argument without being + wrapped with another scalar (e.g. YEAR(birth_date) + 10), no script + ordering was used but instead the underlying field (e.g. birth_date) + was used instead as a performance optimisation. This works correctly if + YEAR() is the only ORDER BY arg but if further args are used as tie + breakers for the ordering wrong results are produced. This is because + 2 rows with the different birth_date but on the same year are not tied + as the underlying ordering is on birth_date and not on the + YEAR(birth_date), and the following ORDER BY args are ignored. + + Remove this optimisation for YEAR() to avoid incorrect results in + such cases. + + As a consequence another bug is revealed: scalar functions on top + of nested fields produce scripted sorting/filtering which is not yet + supported. In such cases no error was thrown but instead all values for + such nested fields were null and were passed to the script implementing + the sorting/filtering, producing incorrect results. + + Detect such cases and throw a validation exception. + + Fixes: #51224 + (cherry picked from commit f41efd6753dc3650a7eabb3e07b02b3b32c5704c) + +commit f0fad5b6221cacfa8b93a00303b2b279c84354f9 +Author: Nhat Nguyen +Date: Thu Jan 30 09:03:10 2020 -0500 + + Deprecate translog retention settings (#51588) (#51638) + + This change deprecates the translog retention settings as they are + effectively ignored since 7.4. + + Relates #50775 + Relates #45473 + +commit 1064009e9d967bd9e81e10f4eb8dd33808ec1762 +Author: Armin Braun +Date: Thu Jan 30 14:27:05 2020 +0100 + + Allow Parallel Snapshot Restore And Delete (#51608) (#51666) + + There is no reason not to allow deletes in parallel to restores + if they're dealing with different snapshots. + A delete will not remove any files related to the snapshot that + is being restored if it is different from the deleted snapshot + because those files will still be referenced by the restoring + snapshot. + Loading RepositoryData concurrently to modifying it is concurrency + safe nowadays as well since the repo generation is tracked in the + cluster state. + + Closes #41463 + +commit ca8601373aef28d1d12135dc16e20e789ed721ec +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 30 14:15:05 2020 +0100 + + [DOCS] Task management API experimental status issue (#51634) + + Add issue reference to documentation. + + Relates #51628 + +commit f7e208237826fe5ce50cb51fde25b5c5d05cba71 +Author: Martijn van Groningen +Date: Thu Jan 30 14:01:30 2020 +0100 + + Backport: unmute rolling upgrade watcher tests and (#51664) + + set watcher logger to debug level. + + These tests haven't run in such a long time, + we first need to get a better picture how/if + these tests fail today. + + Backport of #51478 + See #33185 + +commit 285a167c348522b63099a8ceb81523b04faa2425 +Author: Marios Trivyzas +Date: Thu Jan 30 12:22:03 2020 +0100 + + SQL: Verify Full-Text Search functions not allowed in SELECT (#51568) + + Add a verification that full-text search functions `MATCH()` and `QUERY()` + are not allowed in the SELECT clause, so that a nice error message is + returned to the user early instead of an "ugly" exception. + + Fixes: #47446 + +commit 2e8a2c4baf8e36007cabd9f825716e0c33632ff8 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 30 12:54:55 2020 +0100 + + Fix ActionListener.map exception handling (#50886) (#51642) + + ActionListener.map would call listener.onFailure for exceptions from + listener.onResponse, but this means we could double trigger some + listeners which is generally unexpected. Instead, we should assume that + a listener's onResponse (and onFailure) implementation is responsible + for its own exception handling. + +commit b5106aa59d0f1713d0561ab049f7118984e5ecb8 +Author: Hendrik Muhs +Date: Thu Jan 30 10:08:55 2020 +0100 + + dump audit index to logs for better debugging (#51627) + + The audit index is re-created for every testrun and therefore potential useful debug information + gets lost. This change reads out the audit index and logs the results, which makes them available + for debugging CI issues. + + relates #51549 + +commit f25b6cc2ebac82fcfb6560d84f3310d5deb685d9 +Author: Albert Zaharovits +Date: Thu Jan 30 11:59:11 2020 +0200 + + Add new 'maintenance' index privilege #50643 + + This commit creates a new index privilege named `maintenance`. + The privilege grants the following actions: `refresh`, `flush` (also synced-`flush`), + and `force-merge`. Previously the actions were only under the `manage` privilege + which in some situations was too permissive. + + Co-authored-by: Amir H Movahed + +commit b5a2ee5be22f8ab1eabc522a3315ce2912659a34 +Author: Peter Dyson +Date: Thu Jan 30 17:41:30 2020 +1000 + + [DOCS] Fix minor typo affecting formatting (#51655) + +commit 149b68d8501147dbd137feb36f9fe15f41a9ddca +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 30 08:21:52 2020 +0100 + + [ML] Fix possible race condition starting datafeed (#51646) + + Datafeeds being closed while starting could result in and NPE. This was + handled as any other failure, masking out the NPE. However, this + conflicts with the changes in #50886. + + Related to #50886 and #51302 + +commit 28f2f3dd02a2156af2599bf6122bbf00b69f9db0 +Author: Lisa Cawley +Date: Wed Jan 29 16:52:40 2020 -0800 + + [DOCS] Minor fixes in transform documentation (#51633) + +commit cf5a2269a56a98371ebf16005965e1b92141d483 +Author: Ryan Ernst +Date: Wed Jan 29 16:36:21 2020 -0800 + + Fix stderr to also be captured by log4j (#51569) + + In #50259 we redirected stdout and stderr to log4j, to capture jdk + and external library messages. However, a typo in the method name used + to redirect the stream in java means stdout is currently being + duplicated twice, and stderr not captured. This commit corrects that + mistake. Unfortunately this is at a level that cannot really be tested, + thus we are still missing tests for this behavior. + +commit 4f214d20abeb9f2929279e01483ce1a87e019bc8 +Author: Mark Vieira +Date: Thu Jan 23 15:20:17 2020 -0800 + + Limit max concurrency of test cluster nodes to a function of max workers (#51338) + + + (cherry picked from commit 9a0238c7166e70e467ca61c1353157979fd1598b) + +commit 40f4f2d267290df1361e0aaaeeb75e8e1b28e5ff +Author: Julie Tibshirani +Date: Wed Jan 29 14:36:33 2020 -0800 + + Avoid processing search profile results twice. (#51575) + + Just a small clean-up, not motivated by performance. + +commit 0d21d9e2c5718fc0ab1ecc211674318f54cecc3a +Author: Aleksandr Maus +Date: Wed Jan 29 16:38:52 2020 -0500 + + EQL: Enable QA/rest integration tests for snapshot builds only (#51624) (#51645) + + * Related to #51541: [CI] unknown setting [xpack.eql.enabled] in release-tests + +commit 9dcc3ef7e606af1c8e424a06dd70e05666496bc4 +Author: Julie Tibshirani +Date: Wed Jan 29 12:24:41 2020 -0800 + + Always use one shard in vector REST tests. (#51643) + + This PR tries to address the intermittent vector test failures on 7.x by making + sure we create indices with one shard. + + The fix is based on this theory as to what's happening: + * On 7.x, the default number of shards is 1, but in REST tests we randomly use + 2 in order to cover the multiple shards case. In the failing test run, we use 2 + shards and all documents end up on only one shard. + * During a search, the response from the empty shard doesn't produce + deprecation warnings because we never try to execute the script. If not all + shard responses contain the warning headers, then certain deprecation warnings + can be lost (due to the bug described in #33936). + + Addresses #50716. + Relates to #50061. + +commit fdf74f6ae4574fccc3bd5f74ee67d7ee66d64e9b +Author: Lisa Cawley +Date: Wed Jan 29 08:40:38 2020 -0800 + + [DOCS] Removes beta qualifiers from transform documentation (#51553) + +commit 3f4156e95a674ebdcd3431b3bf45d06cfd0912ed +Author: Lisa Cawley +Date: Wed Jan 29 08:35:02 2020 -0800 + + [DOCS] Adds release highlight for transforms (#51555) + +commit 683170b007e7c9bf227a3ec8e7e993be34a9d0a7 +Author: Przemysław Witek +Date: Wed Jan 29 17:17:19 2020 +0100 + + Increase the number of indexed documents to increase a chance that there are at least 2 training rows. (#51607) (#51615) + +commit 74e36942346c68990a4e284b8b47874a95a6444f +Author: Armin Braun +Date: Wed Jan 29 16:07:30 2020 +0100 + + Optimize GCS Repo Uploads (#51596) (#51618) + + For small uploads (that can still be up to 5MB!) we needlessly + reading the `InputStream` into a BAOS which entailed allocating + the `byte[]` for the stream contents twice (because to `toByteArray` on the BAOS copies). + + Also, for resumeable uploads we were needlessly wrapping the output channel and running each individual write in its own privileged context when we could just wrap the whole upload in a single privileged context. + + Relates #51593 + +commit 078e13b1fd1007bab5e8eb43dc6a1119d727cad8 +Author: James Rodewig +Date: Wed Jan 29 08:45:11 2020 -0500 + + [DOCS] Enable EQL on docs integ tests (#51537) + +commit a7f0c699cfe1f6f70c22f45dac23e8ff84529ed5 +Author: Przemko Robakowski +Date: Wed Jan 29 14:58:23 2020 +0100 + + Fix ignore_missing in CsvProcessor (#51600) (#51609) + + This change fixes inverted logic around ignore_missing in CsvProcessor + +commit 8c7840481434286afee977e5e2fdb90579be9796 +Author: Martijn van Groningen +Date: Wed Jan 29 13:51:42 2020 +0100 + + Only query for watcher history docs with the expected state, + otherwise other history docs can be returned, which may not + have the expected state, and this test only checks the first hit. + + Relates to #30777 + +commit e0e35b7feb3c61faaaab69cfd54b019f8852d9ec +Author: David Roberts +Date: Tue Jan 28 14:32:15 2020 +0000 + + [TEST] Mute TimeSeriesLifecycleActionsIT.testWaitForSnapshotSlmExecutedBefore + + Due to https://github.com/elastic/elasticsearch/issues/50781 + +commit b253af36f3bafe929fc0a91c8f94c330c78588c8 +Author: Martijn van Groningen +Date: Wed Jan 29 12:08:18 2020 +0100 + + The watcher indexing listener didn't handle document level exceptions. (#51466) + + Prior to the change the watcher index listener didn't implement the + `postIndex(ShardId, Engine.Index, Engine.IndexResult)` method. This + caused document level exceptions like VersionConflictEngineException + to be ignored. This commit fixes this. + + The watcher indexing listener did implement the `postIndex(ShardId, Engine.Index, Exception)` + method, but that only handles engine level exceptions. + + This change also unmutes the SmokeTestWatcherTestSuiteIT#testMonitorClusterHealth test again. + + Relates to #32299 + +commit 206c8ac3e991f86f5dc567de3cccd71e8f9a4f98 +Author: Martijn van Groningen +Date: Wed Jan 29 12:15:30 2020 +0100 + + Muted test, see #48381 + +commit d8bd736f8a2366bd39918b16ea7d9f4ad76f9531 +Author: Rory Hunter +Date: Wed Jan 29 10:42:04 2020 +0000 + + Formatting: keep simple if / else on the same line (#51544) + + Backport of #51526. + + Previous the formatter was breaking simple if/else statements (i.e. + without braces) onto separate lines, which could be fragile because the + formatter cannot also introduce braces. Instead, keep such expressions + on the same line. + +commit 7914c1a7342264a7919fe951c8176738ae3d65e4 +Author: Armin Braun +Date: Wed Jan 29 11:06:05 2020 +0100 + + Optimize GCS Mock (#51593) (#51594) + + This test was still very GC heavy in Java 8 runs in particular + which seems to slow down request processing to the point of timeouts + in some runs. + This PR completely removes the large number of O(MB) `byte[]` allocations + that were happening in the mock http handler which cuts the allocation rate + by about a factor of 5 in my local testing for the GC heavy `testSnapshotWithLargeSegmentFiles` + run. + + Closes #51446 + Closes #50754 + +commit 90285ee907b96aed2d77a465fb7ee2571e0e1e18 +Author: Albert Zaharovits +Date: Wed Jan 29 10:48:26 2020 +0200 + + Deprecate timeout.tcp_read AD/LDAP realm setting (#47305) + + The timeout.tcp_read AD/LDAP realm setting, despite the low-level + allusion, controls the time interval the realms wait for a response for + a query (search or bind). If the connection to the server is synchronous + (un-pooled) the response timeout is analogous to the tcp read timeout. + But the tcp read timeout is irrelevant in the common case of a pooled + connection (when a Bind DN is specified). + + The timeout.tcp_read qualifier is hereby deprecated in favor of + timeout.response. + + In addition, the default value for both timeout.tcp_read and + timeout.response is that of timeout.ldap_search, instead of the 5s (but + the default for timeout.ldap_search is still 5s). The + timeout.ldap_search defines the server-controlled timeout of a search + request. There is no practical use case to have a smaller tcp_read + timeout compared to ldap_search (in this case the request would time-out + on the client but continue to be processed on the server). The proposed + change aims to simplify configuration so that the more common + configuration change, adjusting timeout.ldap_search up, has the expected + result (no timeout during searches) without any additional + modifications. + + Closes #46028 + +commit ba3051a50fe0ba84240626128a462c6670ce5a54 +Author: Ioannis Kakavas +Date: Wed Jan 29 08:16:47 2020 +0200 + + Mute Netty4ClientYamlTestSuiteIT in FIPS 140 (#51536) + + rest-api-spec/test/10_basic.yml would check that transport_types is + `netty4` but we run FIPS 140 tests with default distribution and + transport_types is `security4` + +commit 81e7d926f66e2552b274eb85c07f0c3acd4ab49e +Author: Ioannis Kakavas +Date: Wed Jan 29 08:14:38 2020 +0200 + + Add HLRC docs for AuthN and TLS (#51355) (#51551) + + This commit adds examples in our documentation for + + - An HLRC instance authenticating to an elasticsearch cluster using + an elasticsearch token service access token or an API key + - An HLRC instance connecting to an elasticsearch cluster that is + setup for TLS on the HTTP layer when the CA certificate of the + cluster is available either as a PEM file or a keystore + - An HLRC instance connecting to an elasticsearch cluster that + requires client authentication where the client key and certificate + are available in a keystore + + Co-Authored-By: Lisa Cawley + +commit 46166b9b409198875b34d798e8debd5415e428b2 +Author: Mark Vieira +Date: Tue Jan 28 19:50:49 2020 -0800 + + Restore class that is still in use + +commit 7df38210007bdefca74b73984e483e1b7c11f0bd +Author: Mark Vieira +Date: Tue Jan 28 19:38:28 2020 -0800 + + Remove unused classes (#51563) + + + (cherry picked from commit 1d654e8776b5d7235320e03f2d1849a8e0b7da0f) + +commit 316fba0c676d4a0dd801083bbc7c4183b3f8edb7 +Author: Nhat Nguyen +Date: Tue Jan 28 21:37:38 2020 -0500 + + Ensure warm up engine in testTranslogReplayWithFailure + + We need to warm up the engine (i.e., perform an external refresh) before + accessing the external refresh. Note that we refresh externally before + allowing reading from a shard. + + Relates #48605 + Closes #51548 + +commit b080237837221f7c7a53004c32d07b59501b3e4e +Author: Jason Tedor +Date: Tue Jan 28 18:44:06 2020 -0500 + + Ignore virtual ethernet devices that disappear (#51581) + + When checking if a device is up, today we can run into virtual ethernet + devices that disappear while we are in the middle of checking. This + leads to "no such device". This commit addresses such devices by + treating them as not being up, if they are virtual ethernet devices that + disappeared while we were checking. + +commit 3a7192966a2e5654f33ef27cdf76f2614e1e9f5a +Author: Jason Tedor +Date: Tue Jan 28 18:38:02 2020 -0500 + + Check if interface is up for loopback devices only (#51583) + + In the SQL with SSL tests, we need to find the interfaces that are up, + are loopback devices, or have a loopback address. If we check if the + device is up first, we can run into situations where the device is a + virtual ethernet device that might have disappeared between us seeing + the device, and checking if it is up. By first checking if the device is + a loopback device or it has a loopback address, then we can avoid + checking if the device is up except for loopback devices and therefore + we can avoid the disappearing virtual ethernet device problem. + +commit 77f4aafaa2321e5668852373335ce40920ab8c8a +Author: Jim Ferenczi +Date: Tue Jan 28 22:03:04 2020 +0100 + + Expose the logic to cancel task when the rest channel is closed (#51423) + + This commit moves the logic that cancels search requests when the rest channel is closed + to a generic client that can be used by other APIs. This will be useful for any rest action + that wants to cancel the execution of a task if the underlying rest channel is closed by the + client before completion. + + Relates #49931 + Relates #50990 + Relates #50990 + +commit aae93a75780c00f3bc96a25ffae02dc20ad1b127 +Author: Armin Braun +Date: Tue Jan 28 18:33:26 2020 +0100 + + Allow Repository Plugins to Filter Metadata on Create (#51472) (#51542) + + * Allow Repository Plugins to Filter Metadata on Create + + Add a hook that allows repository plugins to filter the repository metadata + before it gets written to the cluster state. + +commit b8adb59e4a908d6394663054a41cb77383be21aa +Author: David Roberts +Date: Tue Jan 28 17:14:36 2020 +0000 + + [TEST] Mute ReloadSecureSettingsIT.testReloadAllNodesWithPasswordWithoutTLSFails + + Due to https://github.com/elastic/elasticsearch/issues/51546 + +commit 89c2834b24963e121aef0c262c59abe243701967 +Author: Gordon Brown +Date: Tue Jan 28 10:01:16 2020 -0700 + + Deprecate creation of dot-prefixed index names except for hidden and system indices (#49959) + + This commit deprecates the creation of dot-prefixed index names (e.g. + .watches) unless they are either 1) a hidden index, or 2) registered by + a plugin that extends SystemIndexPlugin. This is the first step + towards more thorough protections for system indices. + + This commit also modifies several plugins which use dot-prefixed indices + to register indices they own as system indices, and adds a plugin to + register .tasks as a system index. + +commit 139305ffc836ed86af55cc92f79e75acca6ab359 +Author: James Rodewig +Date: Tue Jan 28 10:57:45 2020 -0500 + + [DOCS] Document `indices` cluster stats (#50527) + + Documents the header and `indices` response parameters returned by the + `_cluster/stats` API. + + Co-Authored-By: David Turner + +commit fa212fe60b1f047481377511c1e2614fcd6b209e +Author: Yannick Welsch +Date: Tue Jan 28 15:36:29 2020 +0100 + + Stricter checks of setup and teardown in docs tests (#51430) + + Adds extra checks due to 7.x backport + +commit 1197ebb9e9db22b4495fcbec8748e5372fa46a2b +Author: Jake Landis +Date: Wed Oct 30 17:12:16 2019 -0500 + + Additional assertions for SLM restart tests (#48428) + + Adds the ability to conditionally wipe SLM policies + and ensures that you can read what you wrote across + restarts. + +commit f6686345c98a34638ac0db7eb0d6962ebaf7814a +Author: Yannick Welsch +Date: Tue Jan 28 09:52:24 2020 +0100 + + Avoid unnecessary setup and teardown in docs tests (#51430) + + The docs tests have recently been running much slower than before (see #49753). + + The gist here is that with ILM/SLM we do a lot of unnecessary setup / teardown work on each + test. Compounded with the slightly slower cluster state storage mechanism, this causes the + tests to run much slower. + + In particular, on RAMDisk, docs:check is taking + + ES 7.4: 6:55 minutes + ES master: 16:09 minutes + ES with this commit: 6:52 minutes + + on SSD, docs:check is taking + + ES 7.4: ??? minutes + ES master: 32:20 minutes + ES with this commit: 11:21 minutes + +commit 70e4ae3381c9354a5a29a38b42bc2bb88475683b +Author: James Rodewig +Date: Tue Jan 28 10:33:45 2020 -0500 + + [DOCS] Reformat unique token filter docs (#50748) + + * Updates the description + * Adds analyze, custom analyzer, and custom filter snippets + * Adds parameter documentation + +commit 550254ec7f62f9b20dfa240dafeb3e3c84d285a0 +Author: David Roberts +Date: Tue Jan 28 12:46:00 2020 +0000 + + [ML] Use CSV ingest processor in find_file_structure ingest pipeline (#51492) + + Changes the find_file_structure response to include a CSV + ingest processor in the ingest pipeline it suggests. + + Previously the Kibana file upload functionality parsed CSV + in the browser, but by parsing CSV in the ingest pipeline + it makes the Kibana file upload functionality more easily + interchangable with Filebeat such that the configurations + it creates can more easily be used to import data with the + same structure repeatedly in production. + +commit a8bd4d08e3d08bee91a0d0e9ad9737adeb6e27a4 +Merge: 53e4d1ef071 eb1ed2a35f0 +Author: Aleksandr Maus +Date: Tue Jan 28 09:19:39 2020 -0500 + + Merge branch 'feature/eql_backport' into 7.x + +commit 53e4d1ef071aad39a53e52cfcfe3fdb664bc6de5 +Author: Hendrik Muhs +Date: Tue Jan 28 13:36:23 2020 +0100 + + [Transform] fix TransformRobustnessIT intermittent test failures part 2 (#51523) + + add wait for completion in transform robustness test to avoid occasional test failures during cleanup + + fixes #51347 + +commit 6283648da28e2644ea6a87e3daf69bbe8681dcdf +Author: David Roberts +Date: Tue Jan 28 11:25:44 2020 +0000 + + [TEST] Mute some search.aggregation/10_histogram/date_histogram tests + + Due to https://github.com/elastic/elasticsearch/issues/51525 + +commit d43cd655851eb22fdebc5904e15d05a9756d4d04 +Author: Jason Tedor +Date: Tue Jan 28 05:42:11 2020 -0500 + + Upgrade the bundled JDK to JDK 13.0.2 (#51511) + + This commit upgrades the bundled JDK to JDK 13.0.2, the latest available + patch release of the JDK. + +commit 9efa5be60e3e3b3cd050b11404c9c6d3d7f74822 +Author: William Brafford +Date: Tue Jan 28 05:32:32 2020 -0500 + + Password-protected Keystore Feature Branch PR (#51123) (#51510) + + * Reload secure settings with password (#43197) + + If a password is not set, we assume an empty string to be + compatible with previous behavior. + Only allow the reload to be broadcast to other nodes if TLS is + enabled for the transport layer. + + * Add passphrase support to elasticsearch-keystore (#38498) + + This change adds support for keystore passphrases to all subcommands + of the elasticsearch-keystore cli tool and adds a subcommand for + changing the passphrase of an existing keystore. + The work to read the passphrase in Elasticsearch when + loading, which will be addressed in a different PR. + + Subcommands of elasticsearch-keystore can handle (open and create) + passphrase protected keystores + + When reading a keystore, a user is only prompted for a passphrase + only if the keystore is passphrase protected. + + When creating a keystore, a user is allowed (default behavior) to create one with an + empty passphrase + + Passphrase can be set to be empty when changing/setting it for an + existing keystore + + Relates to: #32691 + Supersedes: #37472 + + * Restore behavior for force parameter (#44847) + + Turns out that the behavior of `-f` for the add and add-file sub + commands where it would also forcibly create the keystore if it + didn't exist, was by design - although undocumented. + This change restores that behavior auto-creating a keystore that + is not password protected if the force flag is used. The force + OptionSpec is moved to the BaseKeyStoreCommand as we will presumably + want to maintain the same behavior in any other command that takes + a force option. + + * Handle pwd protected keystores in all CLI tools (#45289) + + This change ensures that `elasticsearch-setup-passwords` and + `elasticsearch-saml-metadata` can handle a password protected + elasticsearch.keystore. + For setup passwords the user would be prompted to add the + elasticsearch keystore password upon running the tool. There is no + option to pass the password as a parameter as we assume the user is + present in order to enter the desired passwords for the built-in + users. + For saml-metadata, we prompt for the keystore password at all times + even though we'd only need to read something from the keystore when + there is a signing or encryption configuration. + + * Modify docs for setup passwords and saml metadata cli (#45797) + + Adds a sentence in the documentation of `elasticsearch-setup-passwords` + and `elasticsearch-saml-metadata` to describe that users would be + prompted for the keystore's password when running these CLI tools, + when the keystore is password protected. + + Co-Authored-By: Lisa Cawley + + * Elasticsearch keystore passphrase for startup scripts (#44775) + + This commit allows a user to provide a keystore password on Elasticsearch + startup, but only prompts when the keystore exists and is encrypted. + + The entrypoint in Java code is standard input. When the Bootstrap class is + checking for secure keystore settings, it checks whether or not the keystore + is encrypted. If so, we read one line from standard input and use this as the + password. For simplicity's sake, we allow a maximum passphrase length of 128 + characters. (This is an arbitrary limit and could be increased or eliminated. + It is also enforced in the keystore tools, so that a user can't create a + password that's too long to enter at startup.) + + In order to provide a password on standard input, we have to account for four + different ways of starting Elasticsearch: the bash startup script, the Windows + batch startup script, systemd startup, and docker startup. We use wrapper + scripts to reduce systemd and docker to the bash case: in both cases, a + wrapper script can read a passphrase from the filesystem and pass it to the + bash script. + + In order to simplify testing the need for a passphrase, I have added a + has-passwd command to the keystore tool. This command can run silently, and + exit with status 0 when the keystore has a password. It exits with status 1 if + the keystore doesn't exist or exists and is unencrypted. + + A good deal of the code-change in this commit has to do with refactoring + packaging tests to cleanly use the same tests for both the "archive" and the + "package" cases. This required not only moving tests around, but also adding + some convenience methods for an abstraction layer over distribution-specific + commands. + + * Adjust docs for password protected keystore (#45054) + + This commit adds relevant parts in the elasticsearch-keystore + sub-commands reference docs and in the reload secure settings API + doc. + + * Fix failing Keystore Passphrase test for feature branch (#50154) + + One problem with the passphrase-from-file tests, as written, is that + they would leave a SystemD environment variable set when they failed, + and this setting would cause elasticsearch startup to fail for other + tests as well. By using a try-finally, I hope that these tests will fail + more gracefully. + + It appears that our Fedora and Ubuntu environments may be configured to + store journald information under /var rather than under /run, so that it + will persist between boots. Our destructive tests that read from the + journal need to account for this in order to avoid trying to limit the + output we check in tests. + + * Run keystore management tests on docker distros (#50610) + + * Add Docker handling to PackagingTestCase + + Keystore tests need to be able to run in the Docker case. We can do this + by using a DockerShell instead of a plain Shell when Docker is running. + + * Improve ES startup check for docker + + Previously we were checking truncated output for the packaged JDK as + an indication that Elasticsearch had started. With new preliminary + password checks, we might get a false positive from ES keystore + commands, so we have to check specifically that the Elasticsearch + class from the Bootstrap package is what's running. + + * Test password-protected keystore with Docker (#50803) + + This commit adds two tests for the case where we mount a + password-protected keystore into a Docker container and provide a + password via a Docker environment variable. + + We also fix a logging bug where we were logging the identifier for an + array of strings rather than the contents of that array. + + * Add documentation for keystore startup prompting (#50821) + + When a keystore is password-protected, Elasticsearch will prompt at + startup. This commit adds documentation for this prompt for the archive, + systemd, and Docker cases. + + Co-authored-by: Lisa Cawley + + * Warn when unable to upgrade keystore on debian (#51011) + + For Red Hat RPM upgrades, we warn if we can't upgrade the keystore. This + commit brings the same logic to the code for Debian packages. See the + posttrans file for gets executed for RPMs. + + * Restore handling of string input + + Adds tests that were mistakenly removed. One of these tests proved + we were not handling the the stdin (-x) option correctly when no + input was added. This commit restores the original approach of + reading stdin one char at a time until there is no more (-1, \r, \n) + instead of using readline() that might return null + + * Apply spotless reformatting + + * Use '--since' flag to get recent journal messages + + When we get Elasticsearch logs from journald, we want to fetch only log + messages from the last run. There are two reasons for this. First, if + there are many logs, we might get a string that's too large for our + utility methods. Second, when we're looking for a specific message or + error, we almost certainly want to look only at messages from the last + execution. + + Previously, we've been trying to do this by clearing out the physical + files under the journald process. But there seems to be some contention + over these directories: if journald writes a log file in between when + our deletion command deletes the file and when it deletes the log + directory, the deletion will fail. + + It seems to me that we might be able to use journald's "--since" flag to + retrieve only log messages from the last run, and that this might be + less likely to fail due to race conditions in file deletion. + + Unfortunately, it looks as if the "--since" flag has a granularity of + one-second. I've added a two-second sleep to make sure that there's a + sufficient gap between the test that will read from journald and the + test before it. + + * Use new journald wrapper pattern + + * Update version added in secure settings request + + Co-authored-by: Lisa Cawley + Co-authored-by: Ioannis Kakavas + +commit 2239ba8c6ed0997517e1427f7df701e6cc5dbd5a +Author: Hendrik Muhs +Date: Tue Jan 28 11:31:07 2020 +0100 + + [Transform] avoid mapping problems with index templates (#51368) (#51519) + + insert explict mappings for objects in nested output to avoid clashes with index templates + + fixes #51321 + +commit 61663b495e4708382cedc155d956791dd2366416 +Author: Hendrik Muhs +Date: Tue Jan 28 10:09:49 2020 +0100 + + add an integration test using date_nanos as timestamp (#51477) + + add a test for using date_nanos as timestamp field in a continuous transform + +commit bebce4b1909366c6af951a26f6f83e6cf6afe33f +Author: Hendrik Muhs +Date: Tue Jan 28 10:01:01 2020 +0100 + + audit index creation after it the index has been created (#51479) + + moves audit message for index creation after the index has been successfully created. This has + been confusing for a user where index creation failed but audit reported index creation. + +commit 4f3548fbd7ed763221d8bb8c34115e67d51e48ad +Author: Ioannis Kakavas +Date: Tue Jan 28 10:17:35 2020 +0200 + + Disable diagnostic trust manager in tests (#51501) + + This commit sets `xpack.security.ssl.diagnose.trust` to false in all + of our tests when running in FIPS 140 mode and when settings objects + are used to create an instance of the SSLService. This is needed + in 7.x because setting xpack.security.ssl.diagnose.trust to true + wraps SunJSSE TrustManager with our own DiagnosticTrustManager and + this is not allowed when SunJSSE is in FIPS mode. + An alternative would be to set xpack.security.fips.enabled to + true which would also implicitly disable + xpack.security.ssl.diagnose.trust but would have additional effects + (would require that we set PBKDF2 for password hashing algorithm in + all test clusters, would prohibit using JKS keystores in nodes even + if relevant tests have been muted in FIPS mode etc.) + + Relates: #49900 + Resolves: #51268 + +commit 919083decdda7c81ae37d125bf3da2248fc02e7b +Author: Przemko Robakowski +Date: Tue Jan 28 02:12:09 2020 +0100 + + Don't overwrite target field with SetSecurityUserProcessor (#51454) (#51506) + + * Don't overwrite target field with SetSecurityUserProcessor + + This change fix problem with `SetSecurityUserProcessor` which was overwriting + whole target field and not only fields really filled by the processor. + + Closes #51428 + + * Unused imports removed + +commit 92b611ece1a0238d0a31087b7bc7707dbe6e3bb3 +Author: Jason Tedor +Date: Mon Jan 27 16:48:40 2020 -0500 + + Formalize build snapshot (#51484) + + Today we are repeatedly checking if the current build is a snapshot + build or not by reading the system property build.snapshot. This commit + formalizes this by adding a build parameter to indicate whether or not + the current build is a snapshot build. + +commit eb1ed2a35f04f23bd82d32064e55a13d7a0d7e33 +Author: Aleksandr Maus +Date: Mon Jan 27 16:23:36 2020 -0500 + + Compilation fixes for 7.x + +commit 65f49d0bbae6403130dd69443995af0cc8b4b8df +Author: James Rodewig +Date: Mon Jan 27 16:03:23 2020 -0500 + + [DOCS] Add top-level EQL docs page. Adds EQL requirements page. (#51334) + + * Creates a top-level page for EQL in the ES reference. + This page contains a high-level introduction and will include a nav for other EQL docs pages as they're built. + + * Creates a requirements page. + This page outlines the fields needed to use EQL in ES. + +commit fc2d875c9f6cfb6fa2f106e88ada3ffb2630b1c1 +Author: Tal Levy +Date: Mon Jan 27 12:14:40 2020 -0800 + + Fix geogrid with bounds test edge cases (#51118) + + This commit modifies the bounding box for geogrid unit tests + to only consider bounding boxes that have significant longitudinal + width and whose coordinates are normalized to quantized space + + Closes #51103. + +commit d8f1735e398628a20328eeb95cbf0334a83521fd +Author: Aleksandr Maus +Date: Thu Jan 23 19:50:50 2020 -0500 + + Add xpack.eql.enabled feature flag, disabled by default. Enabled only for integration tests. (#51370) + + Related to https://github.com/elastic/elasticsearch/issues/49581 + +commit d049de5b728b752fd1dca50eae880c3e806aac25 +Author: Costin Leau +Date: Tue Jan 14 09:44:51 2020 +0200 + + EQL: import QL into EQL (#50904) + + Link QL into the new build file + Remove duplicate classes and use the new ql package + Update Exception hierarchy on top of QlException + +commit c1844114567b9ebbfbc5a8a65f9050178773678e +Author: Igor Motov +Date: Mon Jan 13 07:53:36 2020 -1000 + + EQL: Replace EqlSearchResponse.Hits parser with ObjectParser (#50925) + + Replaces the existing hand-build Hits parser with a + ConstructingObjectParser version. + + Relates to #49581 + +commit 88cc30c0d8df8d6782381a03ccd02118e78615fa +Author: Igor Motov +Date: Mon Jan 13 05:06:30 2020 -1000 + + EQL: Remove list classes from EqlSearchResponse (#50870) + + Removes unnecessary classes from EqlSearchResponse that just represent + lists of other elements. + + Relates to #49581 + +commit d715176c001ff9128aa2170ab46b9abbfdc29970 +Author: Aleksandr Maus +Date: Fri Jan 10 13:39:05 2020 -0500 + + Add more Eql REST API validation integration tests, clean up request implementation (#50822) + +commit 628083183fefcba507e44d00f06880821d257bd2 +Author: Igor Motov +Date: Thu Jan 9 09:59:41 2020 -1000 + + EQL: Make EqlSearchResponse immutable (#50810) + + Refactors EqlSearchResponse to make it immutable + + Relates to #49581 + +commit 31d2d01e25c82f0d2cc5f558411c733c9748c79f +Author: Aleksandr Maus +Date: Mon Jan 6 17:56:13 2020 -0500 + + Correct search_after handling (#50629) + +commit 79875ce4d9b9d8cc8564dc568035a13fd9be6ff0 +Author: Aleksandr Maus +Date: Fri Jan 3 14:14:41 2020 -0500 + + Initial EQL rest API implementation (#49768) + +commit 10a16d15d166f9b58d7133745f184564d01cd935 +Author: Costin Leau +Date: Mon Nov 11 16:32:17 2019 +0200 + + Add draft EQL grammar and expression tree + +commit e22f5010188b688b6a1ebd6efd8978ee6c3c4a5f +Author: Costin Leau +Date: Mon Jan 27 22:03:58 2020 +0200 + + QL: Backport project to 7.x (#51497) + + * Introduce reusable QL plugin for SQL and EQL (#50815) + + Extract reusable functionality from SQL into its own dedicated project QL. + Implemented as a plugin, it provides common components across SQL and the upcoming EQL. + + While this commit is fairly large, for the most part it's just a big file move from sql package to the newly introduced ql. + + (cherry picked from commit ec1ac0d463bfa12a02c8174afbcdd6984345e8b4) + + * SQL: Fix incomplete registration of geo NamedWritables + + (cherry picked from commit e295763686f9592976e551e504fdad1d2a3a566d) + + * QL: Extend NodeSubclass to read classes from jars (#50866) + + As the test classes are spread across more than one project, the Gradle + classpath contains not just folders but also jars. + This commit allows the test class to explore the archive content and + load matching classes from said source. + + (cherry picked from commit 25ad74928afcbf286dc58f7d430491b0af662f04) + + * QL: Remove implicit conversion inside Literal (#50962) + + Literal constructor makes an implicit conversion for each value given + which turns out has some subtle side-effects. + Improve MathProcessors to preserve numeric type where possible + Fix bug on issue compatibility between date and intervals + Preserve the source when folding inside the Optimizer + + (cherry picked from commit 9b73e225b0aa07a23859550fb117bae571a2b672) + + * QL: Refactor DataType for pluggability (#51328) + + Change DataType from enum to class + Break DataType enums into QL (default) and SQL types + Make data type conversion pluggable so that new types can be introduced + + As part of the process: + - static type conversion in QL package (such as Literal) has been + removed + - several utility classes have been broken into base (QL) and extended + (SQL) parts based on type awareness + - operators (+,-,/,*) are + - due to extensibility, serialization of arithmetic operation has been + slightly changed and pushed down to the operator executor itself + + (cherry picked from commit aebda81b30e1563b877a8896309fd50633e0b663) + + * Compilation fixes for 7.x + +commit 3eae601b0b471dd0556a5c5b1a1ccce5d1f14aa9 +Author: Rafael Acevedo +Date: Mon Jan 27 16:24:47 2020 -0300 + + Upgrade gradle to 6.1.1 (#51460) + +commit 6ee1baf2ed476397f5123055c4d43c0c2731da9b +Author: Ryan Ernst +Date: Mon Jan 27 10:49:01 2020 -0800 + + Migrate cron eval bats test to java (#50940) (#51007) + + This commit migrates the simple test of the cron eval tool from bats to + java packaging tests. + + relates #46005 + +commit 4ff314a9d5715e97bdc426f4307c6ae0179d9221 +Author: Nik Everett +Date: Mon Jan 27 13:40:54 2020 -0500 + + Begin moving date_histogram to offset rounding (take two) (#51271) (#51495) + + We added a new rounding in #50609 that handles offsets to the start and + end of the rounding so that we could support `offset` in the `composite` + aggregation. This starts moving `date_histogram` to that new offset. + + This is a redo of #50873 with more integration tests. + + This reverts commit d114c9db3e1d1a766f9f48f846eed0466125ce83. + +commit faf6bb27be6b5e4cbf1029786f9534b1b915f0ee +Author: Nik Everett +Date: Mon Jan 27 12:44:48 2020 -0500 + + Support time_zone on composite's date_histogram (#51172) (#51491) + + We've been parsing the `time_zone` parameter on `date_hitogram` for a + while but it hasn't *done* anything. This wires it up. + + Closes #45199 + Inspired by #45200 + +commit 3c223ceea19ebe1a9bfbfdc0edd2c0abda669f60 +Author: David Roberts +Date: Mon Jan 27 17:05:59 2020 +0000 + + [ML] Fix 2 digit year regex in find_file_structure (#51469) + + The DATE and DATESTAMP Grok patterns match 2 digit years + as well as 4 digit years. The pattern determination in + find_file_structure worked correctly in this case, but + the regex used to create a multi-line start pattern was + assuming a 4 digit year. Also, the quick rule-out + patterns did not always correctly consider 2 digit years, + meaning that detection was inconsistent. + + This change fixes both problems, and also extends the + tests for DATE and DATESTAMP to check both 2 and 4 digit + years. + +commit 8559ff7cee6367dcdd8df781272bc98c7f073c5d +Author: Benjamin Trent +Date: Mon Jan 27 12:12:34 2020 -0500 + + [ML][Inference] fixing pattern compilation + unnecessary string copy (#51483) (#51487) + +commit 40bd271f5398c78f063b928e25fe8a3728a4fdf7 +Author: Jason Tedor +Date: Mon Jan 27 09:48:40 2020 -0500 + + Enable autoscaling in snapshot docs tests (#51474) + + This commit enables autoscaling in docs tests based on snapshot + builds. This is done so that when the API docs are added to the docs, + then the cluster will have been started to enable autoscaling so these + APIs are available for testing. + +commit 8b851bfc33715ff55ad42effc9d6a28cc367914c +Author: Martijn van Groningen +Date: Mon Jan 27 14:41:15 2020 +0100 + + Removed more unchecked suppress warnings. + + See #48381 + +commit 23b65390aba785afc6eb224cf1937304b0762500 +Author: James Rodewig +Date: Mon Jan 27 08:41:05 2020 -0500 + + [DOCS] Add response snippets to 'Testing analyzers' page (#51427) + + Adds response snippets to the `POST _analyze` snippets in the 'Testing + analyzers' page. + + Co-authored-by: Emmanuel DEMEY + +commit 716904fab7bee5deec83a3707fce2d74a5daa052 +Author: Martijn van Groningen +Date: Mon Jan 27 14:09:13 2020 +0100 + + Unmuted test with more logging and + removed unchecked suppress warnings. + + See #48381 + +commit b233e9301490d27d3f93c1dd14032aec77aa0ae0 +Author: Hendrik Muhs +Date: Mon Jan 27 14:04:57 2020 +0100 + + [Transform] refactor naming leftovers and apply code formating (#51465) (#51470) + + refactor renaming leftovers: "data frame transform" to "transforms", touch only internals (variable + names, non-public API's, doc strings, ...) and apply code-formatting (spotless). No logical changes. + +commit 2eeea21d84978e47f173c9421a19dec9aa92b474 +Author: Armin Braun +Date: Mon Jan 27 13:28:17 2020 +0100 + + Use Consistent ClusterState throughout Snapshot API Calls (#51464) (#51471) + + We shouldn't be using potentially changing versions of the cluster state + when answering a snapshot status API call by calling `SnapshotService#currentSnapshots` multiple times (each time using `ClusterService#state` under the hood) but instead pass down the state from the transport action. + Having these API behave more in a more deterministic way will make it easier to use them once parallel repository operations + are introduced. + +commit 9085024e1d36285579a5368df017f764347ea41a +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jan 27 12:40:27 2020 +0100 + + Disable reindex against 0.90 on mac (#51449) + + We still test remote reindex against version 0.90. This failed on mac a + few times and rather than spend time investigating this, we no longer + test remote reindex against 0.90 on mac. + + Closes #51202 + +commit 49bde5d286c1cc8cf7d144bf0685b6c23915de74 +Author: David Turner +Date: Mon Jan 27 10:50:10 2020 +0000 + + Remove DEBUG-level default logging from actions (#51459) + + In 2bb31fe (v0.6.0!) we added DEBUG-level logging to the default config of + action loggers "for easier debugging". This change to the default config lives + on to this day. It does not obviously make debugging any easier any more, but + it does result in a good deal of log noise sometimes. This commit removes this + special case from the default config. + + Closes #51198 + +commit 977cce002ea85ccdcfbd76d2457fdb824da64fa2 +Author: Andrei Dan +Date: Mon Jan 27 10:40:40 2020 +0000 + + Preserve slm-history-ilm-policy between test runs (#51442) (#51468) + + (cherry picked from commit 4e95c8a94fa700d44ac31ef17547512748ab1885) + Signed-off-by: Andrei Dan + +commit d872db278a93b4e7a7d5021114eed39d99941039 +Author: Andrei Dan +Date: Mon Jan 27 10:40:29 2020 +0000 + + Fix TimeSeriesLifecycleActionsIT.testShrinkAction (#51431) (#51467) + + * Fix TimeSeriesLifecycleActionsIT.testShrinkAction + + Shrinking a 6 shard index to 3 shards can be quite time consuming and + assertBusy probes the conditions at exponentially growing intervals. + + This separates the one assertion that was used for all the conditions + into multiple assertBusy statements and increases the timeout for waiting + for the shrink to complete. + + * Allow more time for shrink to complete + + This commit allows more time for the shrink operation to complete in + testRetryFailedShrinkAction (separating the assertBusy calls too) and + testMoveToRolloverStep. + + * Shrink to no more than 2 shards in tests + + (cherry picked from commit 5fe780148fa3536915d61475b087896a5b9ace82) + Signed-off-by: Andrei Dan + +commit d289c1d5f12d0fc1ffe5900d9c890f7af0a741a3 +Author: Martijn van Groningen +Date: Mon Jan 27 10:38:03 2020 +0100 + + Wrong bug url in @AwaitsFix + + See #48381 + +commit e253b7e73d8f81e3c0772655857309d6dace0072 +Author: Martijn van Groningen +Date: Mon Jan 27 10:31:47 2020 +0100 + + Retry response exceptions in the test. + + Relates to #30777 + +commit 7e0f73e03507341ccf92499fc1d2e70c50f7438f +Author: Martijn van Groningen +Date: Fri Jan 24 15:29:32 2020 +0100 + + Muted watcher bwc restart test #30777 + +commit ee202a642f663eb16f5fccfc984f6db02aa67836 +Author: Ioannis Kakavas +Date: Mon Jan 27 11:14:52 2020 +0200 + + Enable tests in FIPS 140 in JDK 11 (#49485) + + This change changes the way to run our test suites in + JVMs configured in FIPS 140 approved mode. It does so by: + + - Configuring any given runtime Java in FIPS mode with the bundled + policy and security properties files, setting the system + properties java.security.properties and java.security.policy + with the == operator that overrides the default JVM properties + and policy. + + - When runtime java is 11 and higher, using BouncyCastle FIPS + Cryptographic provider and BCJSSE in FIPS mode. These are + used as testRuntime dependencies for unit + tests and internal clusters, and copied (relevant jars) + explicitly to the lib directory for testclusters used in REST tests + + - When runtime java is 8, using BouncyCastle FIPS + Cryptographic provider and SunJSSE in FIPS mode. + + Running the tests in FIPS 140 approved mode doesn't require an + additional configuration either in CI workers or locally and is + controlled by specifying -Dtests.fips.enabled=true + +commit dd3e2f1e189c2f6314d84640544467133887ed95 +Author: Przemysław Witek +Date: Mon Jan 27 10:13:02 2020 +0100 + + [7.x] Update quantiles document in the index the document belongs to (#51135) (#51415) + +commit fbfc5a327c9f53c8d07e5bda7af11a4942213940 +Author: Ryan Ernst +Date: Sat Jan 25 14:13:47 2020 -0800 + + Fix checkstyle and test for logging output stream + + Commit a156629b didn't quite fix the botched backport. This commit does. + +commit a156629b4ad3b965dfb32039b9ca5071937b927d +Author: Ryan Ernst +Date: Sat Jan 25 11:13:33 2020 -0800 + + Fix backport compilation + + Java 8 doesn't have a PrintStream ctor which takes a Charset object, + only a charset name. This fixes that. + +commit a564cac7ba770c09def6e3251f6995d4b90c8c55 +Author: Ryan Ernst +Date: Sat Jan 25 10:59:31 2020 -0800 + + Capture stdout and stderr to log4j log (#50259) + + This commit overrides the stdout and stderr print streams to be + redirected to the main elasticsearch.log file. While the Elasticsearch + project ensures stdout and stderr are not written to, the jdk or 3rd + party libs may do this, which can be unexepected for users used to + looking the elasticsearch log. + + closes #50156 + +commit 87b926aa22be1f59a7b4fe6624102bfc40b96697 +Author: Ryan Ernst +Date: Sat Jan 25 10:49:38 2020 -0800 + + Fix git info within VM for worktrees (#51119) + + If a worktree is used, say for 7.x, and packaging tests are run, the + build within the VM will fail due to the parent checkout not being + accessible. This is because the path of the worktree is for the host + systtem, not the VM. This commit makes the git info unknown, just as if + the .git directory did not exist. + +commit 3e3673b5188e8ade9593765f6f9639ffbd1279f2 +Author: Armin Braun +Date: Sat Jan 25 16:19:49 2020 +0100 + + Fix ByteBuf Leak in Nio HTTP Tests (#51444) (#51457) + + It is the job of the http server transport to release the request in the handler + but the mock fails to do so since we never override `incomingRequest`. + +commit fbec19c0228551ac3929375cb949a36c21d4b2b0 +Author: Przemko Robakowski +Date: Sat Jan 25 01:19:55 2020 +0100 + + Centralize mocks initialization in ILM steps tests (#51384) (#51453) + + * Centralize mocks initialization in ILM steps tests + + This change centralizes initialization of `Client`, `AdminClient` + and `IndicesAdminClient` for all classes extending `AbstractStepTestCase`. + This removes a lot of code duplication and make it easier to write tests. + This also removes need for `AsyncActionStep#setClient` + + * Unused imports removed + + * Added missed tests + + * Fix OpenFollowerIndexStepTests + +commit 8560847dd96d841e735c7b7af8e7929255ba1393 +Author: Lee Hinman +Date: Fri Jan 24 14:30:42 2020 -0700 + + [7.x] Check all snapshots in SnapshotLifecycleRestIT.testFullP… (#51448) + + * Check all snapshots in SnapshotLifecycleRestIT.testFullPolicy + + Rather than check the first returned snapshot for a snapshot starting with `snap-` in + SnapshotLifecycleRestIT.testFullPolicy, this commit changes the test to find any snapshots starting + with `snap-`. + + In the event that there are no snapshots (the failure case), this also exposes the full results map + so we can diagnose why a failure occurred. + + Relates to #50358 + + * Use a more imperative style for checking + +commit 2425a1a890877ea198bd050f25447d1837667d9d +Author: William Brafford +Date: Fri Jan 24 16:23:18 2020 -0500 + + Use '--cursor-after' flag to get recent journal messages (#51366) (#51445) + + When we get Elasticsearch logs from journald, we want to fetch only log + messages from the last run. There are two reasons for this. First, if + there are many logs, we might get a string that's too large for our + utility methods. Second, when we're looking for a specific message or + error, we almost certainly want to look only at messages from the last + execution. + + Previously, we've been trying to do this by clearing out the physical + files under the journald process. But there seems to be some contention + over these directories: if journald writes a log file in between when + our deletion command deletes the file and when it deletes the log + directory, the deletion will fail. + + Instead, we can use the cursor capablity of journald to retrieve journal + entries that occur only after a certain cursor. This avoids any effort + to interfere with the underlying file operations of journald. + +commit ef94a5863ab886f9dd21786354ffa6e2a05b4b7b +Author: Armin Braun +Date: Fri Jan 24 20:33:45 2020 +0100 + + Fix RareClusterStateIT Cancelling Publication too Early (#51429) (#51434) + + Wait for the cluster to have settled down and have the same accepted version on all nodes before + executing and cancelling request so that a slow CS accept on one node doesn't make it fall behind + and then get sent the full CS because of the diff-version mismatch, breaking the mechanics of this test. + + Closes #51308 + +commit af1ff52e70117dbfff8a893fe01f75e7a1024c8d +Author: Armin Braun +Date: Fri Jan 24 20:33:13 2020 +0100 + + Fix TransportMasterNodeAction not Retrying NodeClosedException (#51325) (#51437) + + Added node closed exception to the retryable remote exceptions as it's possible to run into this exception instead of a connect exception when the master node is just shutting down but still responding to requests. + +commit ef0a93354144fb01fd57321892ce3b3f97f15f2b +Author: Mark Vieira +Date: Fri Jan 24 11:00:14 2020 -0800 + + Tag "nested" Gradle build scans (#51440) + +commit f38eae7abc56087e88dac656d3024f9c3148908a +Author: Jason Tedor +Date: Fri Jan 24 13:10:59 2020 -0500 + + Install plugins in a single transaction in tests (#51433) + + When building clusters for integration tests, today we install plugins + sequentially. We recently introduced the ability to install plugins in a + single invocation of the install plugin command. Using this can save + substantial time starting up JVMs. This commit changes the build + infrastructure to install multiple plugins at once when building + clusters for integration tests. + + For the docs integration tests in particular, where we install many + plugins, this change makes a substantial difference. On my laptop, prior + to this change, installing the plugins sequentially took 115 + seconds. After this change, it takes 14 seconds. + +commit bdb8b6aa0d6a0592b5fce20637008718a5045252 +Author: Lee Hinman +Date: Fri Jan 24 11:05:19 2020 -0700 + + [7.x] Separate aliases used for tests in TimeSeriesLifecycleAc… (#51432) + + * Separate aliases used for tests in TimeSeriesLifecycleActionsIT + + This is related to #51375 and hopes to help illuminate why some of those tests are failing. This + commit switches the aliases used in the test to use a random alias name every time (since there were + some complaints in the tests about aliases having more than one write index). With this we hope to + determine the actual cause of the failure in the test. + + This also adds additional information to the exception returned when calling move-to-step with the + incorrect current step. + + * Fix rest test + +commit 931b22349f3f993f49ac18c3c4b392b740f96add +Author: Lisa Cawley +Date: Fri Jan 24 09:56:51 2020 -0800 + + [DOCS] Adds http to elasticsearch-certutil command reference (#51188) + +commit f0d8c785e324d3d61057f17ea0bac0a45fd92b41 +Author: Armin Braun +Date: Fri Jan 24 18:20:47 2020 +0100 + + Fix Inconsistent Shard Failure Count in Failed Snapshots (#51416) (#51426) + + * Fix Inconsistent Shard Failure Count in Failed Snapshots + + This fix was necessary to allow for the below test enhancement: + We were not adding shard failure entries to a failed snapshot for those + snapshot entries that were never attempted because the snapshot failed during + the init stage and wasn't partial. This caused the never attempted snapshots + to be counted towards the successful shard count which seems wrong and + broke repository consistency tests. + + Also, this change adjusts snapshot resiliency tests to run another snapshot + at the end of each test run to guarantee a correct `index.latest` blob exists + after each run. + + Closes #47550 + +commit bf53ca33804b0796ad4ae81841935107094e68fc +Author: Benjamin Trent +Date: Fri Jan 24 11:54:22 2020 -0500 + + [7.x] [ML] Add _cat/ml/anomaly_detectors API (#51364) (#51408) + + [ML] Add _cat/ml/anomaly_detectors API (#51364) + +commit a29deecbda90191049a90d2c164f1a20ded0a99d +Author: Mayya Sharipova +Date: Fri Jan 24 11:05:42 2020 -0500 + + Revert "Make it clear this is boost at index time (#51390)" + + This reverts commit 3d5238bd9571faa6e6947365c35e117df68b8fd6. + +commit 3d5238bd9571faa6e6947365c35e117df68b8fd6 +Author: Jonas F. Henriksen +Date: Fri Jan 24 16:36:06 2020 +0100 + + Make it clear this is boost at index time (#51390) + + The way it was originally written, it sounds like + we are boosting at query time. + Of course, the effect is at query time, + but the point here is that boosting is done at index time + +commit fc994d9ce1981346720e83d1452a8df6d6768e77 +Author: Benjamin Trent +Date: Fri Jan 24 09:29:12 2020 -0500 + + [ML][Inference] Adds validations for model PUT (#51376) (#51409) + + Adds validations making sure that + + * `input.field_names` is not empty + * `ensemble.trained_models` is not empty + * `tree.feature_names` is not empty + + closes https://github.com/elastic/elasticsearch/issues/51354 + +commit d177747f660ea7ec2b38db62d37b6c1dc46bb1ef +Author: Hendrik Muhs +Date: Fri Jan 24 15:18:19 2020 +0100 + + fix TransformRobustnessIT intermittent test failures + + ensure the cluster is not in some intermediate state when cleaning up. + + fixes #51347 + +commit d0fbf71314e1020534f465af589462f88f1f11da +Author: Przemyslaw Gomulka +Date: Fri Jan 24 15:02:20 2020 +0100 + + Reenable the test with the correct version (#51396) + + The Iso week calculations were enabled in 7.7 (previously incorrectly test was allowed to run against 7.6) + +commit 36b460060c99ccc553d7559c73b296fbf578a356 +Author: Martijn van Groningen +Date: Fri Jan 24 14:37:26 2020 +0100 + + Unmuted watcher security smoke tests on 7 dot x branch. + + Also removed the usage of types in watcher's index action and + added more logging in case this test fails again. + + Relates to #30777 + +commit 7af04741011a7a644f1837c72ba352bda671bcdd +Author: Martijn van Groningen +Date: Thu Jan 16 10:29:45 2020 +0100 + + Add more logging when failing watch history entry fails. (#50931) + + Relates to #30777 + +commit 76660a5a4f49eb0e96e3ec321d070c3c15f046ea +Author: Benjamin Trent +Date: Fri Jan 24 08:26:58 2020 -0500 + + [7.x] [ML][Inference] add tags url param to GET (#51330) (#51404) + + * [ML][Inference] add tags url param to GET (#51330) + + Adds a new URL parameter, `tags` to the GET _ml/inference/ endpoint. + + This parameter allows the list of models to be further reduced to those who contain all the provided tags. + +commit d3078c5b404ab00b34e2fbc5d6162f6358780370 +Author: Martijn van Groningen +Date: Mon Jan 6 10:47:52 2020 +0100 + + Re-enable FullClusterRestartIT#testWatcher test (#50463) + + Previously this test failed waiting for yellow: + https://gradle-enterprise.elastic.co/s/fv55holsa36tg/console-log#L2676 + + Oddly cluster health returned red status, but there were no unassigned, relocating or initializing shards. + Placed the waiting for green in a try-catch block, so that when this fails again then cluster state gets printed. + + Relates to #48381 + +commit 53ac28e3985194caae57e5191e92b0c6404a2ee2 +Author: Martijn van Groningen +Date: Fri Jan 24 14:02:27 2020 +0100 + + Update smoke test watcher test suite with the changes in master branch. + + Relates to #32299 + +commit 8bdf654cc71024b56297d35c80eb584e6d326c9e +Author: István Zoltán Szabó +Date: Fri Jan 24 13:31:44 2020 +0100 + + [DOCS] Refines description. (#51400) + +commit ded7407b4dca05afde8795447d8f3c12d70a7c9b +Author: Hendrik Muhs +Date: Fri Jan 24 12:17:32 2020 +0100 + + [Transform] Adapt tests for error message to 7.x format + + adapt messages to 7.x format (#51398) + + fixes #51360 + +commit 2f7c2401841fce09645f2ceebc4ce9af4d2f2b28 +Author: Andrei Dan +Date: Fri Jan 24 10:53:37 2020 +0000 + + [7.x] Use ESSingleNodeTestCase instead of ESIntegTestCase (#51345) (#51346) + + * Use ESSingleNodeTestCase instead of ESIntegTestCase (#51345) + + (cherry picked from commit abcf1c41faf05a0b0196fb06e57c3de8c3d67688) + Signed-off-by: Andrei Dan + +commit c29b235a5a45bc09121132f93cc8603fe12cc069 +Author: Armin Braun +Date: Fri Jan 24 11:23:16 2020 +0100 + + Stop Copying Bulk HTTP Requests in NIO Networking (#49819) (#51393) + + Same as #44564 but for NIO. + +commit 8703b885c25fd40241a1268bfa5b7dc2d436dde6 +Author: Przemysław Witek +Date: Fri Jan 24 11:10:54 2020 +0100 + + Move TupleMatchers class to org.elasticsearch.test.hamcrest package (#51359) (#51395) + +commit 40e7a826fc06d424e39eb245e56d97833cc76877 +Author: David Turner +Date: Fri Jan 24 08:58:18 2020 +0000 + + Allow decimal max_task_wait_time in docs (#51352) + + The regex for the response to `GET _cat/health?v` in `getting-started.asciidoc` + requires `max_task_wait_time` to match `(-|\\d+(micros|ms|s))`, which doesn't + match times such as `3.9ms` that contain a decimal point. This commit adjusts + the regex to match times formatted like this too. + + Fixes #47537 + +commit 0981a469ae3b19b8ece9146a657b4b835fc23c1f +Author: Tim Vernum +Date: Fri Jan 24 19:35:07 2020 +1100 + + Preserve ApiKey credentials for async verification (#51389) + + The ApiKeyService would aggressively "close" ApiKeyCredentials objects + during processing. However, under rare circumstances, the verfication + of the secret key would be performed asychronously and may need access + to the SecureString after it had been closed by the caller. + + The trigger for this would be if the cache already held a Future for + that ApiKey, but the future was not yet complete. In this case the + verification of the secret key would take place asynchronously on the + generic thread pool. + + This commit moves the "close" of the credentials to the body of the + listener so that it only occurs after key verification is complete. + + Backport of: #51244 + +commit d46e8c3f7fad184d7261db05e31838f45523c933 +Author: Hendrik Muhs +Date: Fri Jan 24 09:04:06 2020 +0100 + + [Transform] disallow dotted fieldnames (#51369) + + adds field validation to disallow output field names starting and/or ending with a '.'. Avoids + indexing/mapping problems when starting the transform. + +commit 3443d69883b45a28679221b4d53f80cb1b320521 +Author: Dimitris Athanasiou +Date: Fri Jan 24 09:51:48 2020 +0200 + + [7.x][ML] Rename DataFrameAnalyticsIndex to DestinationIndex (#51353) (#51356) + + As we prepare to introduce a new index for storing additional + information about data frame analytics jobs (e.g. intrumentation), + renaming this class to `DestinationIndex` better captures what it does + and leaves its prior name available for a more suitable use. + + Backport of #51353 + + Co-authored-by: Elastic Machine + +commit 144e037941f835c5749298d5d30f0d0986851cdc +Author: Ryan Ernst +Date: Thu Jan 23 15:48:34 2020 -0800 + + Ignore order of indices in hidden indices test (#51383) + + The order indices are returned in in the metadata is not guaranteed. + This commit accounts for any possible ordering in assertions about + hidden indices. + + closes #51340 + +commit 31747de2a2f287bcaf58720523a5ce1f843d96d5 +Author: Lee Hinman +Date: Thu Jan 23 15:46:22 2020 -0700 + + Fix testHashcodeAndEquals mutation for WaitForSnapshotStep (#51379) + + This fixes the test failure, it was randomness returning the same policy + rather than a new one. Switched to use `randomValueOtherThan`. + + Resolves #51377 + +commit 9a7124027b37f117c6fbfc39f33ec721aafb587d +Author: Mark Vieira +Date: Thu Jan 23 15:03:14 2020 -0800 + + Add 7.6.0 staged version to BWC testing matrix + +commit f86de2a9cbaa7303291533ac6e646ec0a2e89364 +Author: Mark Vieira +Date: Thu Jan 23 14:54:57 2020 -0800 + + Always test against default distribution when in a FIPS JVM (#51273) (#51333) + +commit 1ca5dd13defa16a18e6374deb6f42d85f0d98309 +Author: Nhat Nguyen +Date: Thu Jan 23 17:20:57 2020 -0500 + + Flush instead of synced-flush inactive shards (#51365) + + If all nodes are on 7.6, we prefer to perform a normal flush + instead of synced flush when a shard becomes inactive. + + Backport of #49126 + +commit 072203cba8cfc45d5aab5fed8084c041a20f2412 +Author: Nhat Nguyen +Date: Thu Jan 23 16:31:47 2020 -0500 + + Clean soft-deletes setting in ccr tests (#51113) (#51372) + + We no longer need to explicitly enable soft-deletes in CCR tests. + + Relates #50775 + Backport of #51113 + +commit 93f60f4c012385eda72bdb15c8cf42879d1d5de8 +Author: Zachary Tong +Date: Thu Jan 23 13:24:25 2020 -0500 + + Mute "Date aggregation per day of week" yaml test + + Tracking issue: https://github.com/elastic/elasticsearch/issues/51367 + +commit 2e314a133f5b0813320e381cb786b0a11ea3c80d +Author: Zachary Tong +Date: Thu Jan 23 12:53:02 2020 -0500 + + Mute TransformTaskFailedStateIT#testStartFailedTransform + + Tracking issue: https://github.com/elastic/elasticsearch/issues/51360 + +commit 0152c4072456bdd0b3e5c209c489e5d3a13b4e5a +Author: David Turner +Date: Thu Jan 23 17:27:59 2020 +0000 + + Log when probe succeeds but full connection fails (#51304) (#51357) + + It is permitted for nodes to accept transport connections at addresses other + than their publish address, which allows a good deal of flexibility when + configuring discovery. However, it is not unusual for users to misconfigure + nodes to pick a publish address which is inaccessible to other nodes. We see + this happen a lot if the nodes are on different networks separated by a proxy, + or if the nodes are running in Docker with the wrong kind of network config. + + In this case we offer no useful feedback to the user unless they enable + TRACE-level logs. It's particularly tricky to diagnose because if we test + connectivity between the nodes (using their discovery addresses) then all will + appear well. + + This commit adds a WARN-level log if this kind of misconfiguration is detected: + the probe connection has succeeded (to indicate that we are really talking to a + healthy Elasticsearch node) but the followup connection attempt fails. + + It also tidies up some loose ends in `HandshakingTransportAddressConnector`, + removing some TODOs that need not be completed, and registering its + accidentally-unregistered timeout settings. + +commit feb2a2576109195cf699313e7bf37be926bd811c +Author: Zachary Tong +Date: Thu Jan 23 11:58:36 2020 -0500 + + Mute TransformTaskFailedStateIT#testForceStopFailedTransform + + Tracking issue: https://github.com/elastic/elasticsearch/issues/51360 + +commit d279b462f79e92647cab0655dbde360811fde77e +Author: Ioannis Kakavas +Date: Thu Jan 23 17:45:55 2020 +0200 + + Fix flaky testCreateApiKey test (#51223) (#51349) + + API Key expiration value has millisecond precision as we use + {@link Instant#toEpoqueMilli()} when creating the API key + document. + It could often happen that `Instant.now()` Instant in the testCreateApiKey + was close enough to the ApiKeyService's `clock.instant()` Instant, + when the nanos were removed from the latter ( due to the call + to `toEpoqueMilli()` ) the result of comparing these two Instants + was a few nanos short of a 7 days. + + Resolves: #47958 + +commit 3553f68f5ac4beb4977b5a1185350b0b365d2110 +Author: Hendrik Muhs +Date: Thu Jan 23 16:16:27 2020 +0100 + + [Transform] Handle permanent bulk indexing errors (#51307) + + check bulk indexing error for permanent problems and ensure the state goes into failed instead of + retry. Corrects the stats API to show the real error and avoids excessive audit logging. + + fixes #50122 + +commit 84664e8d600fd2701dab52b42404b59757247157 +Author: Przemko Robakowski +Date: Thu Jan 23 15:28:13 2020 +0100 + + Expose master timeout for ILM actions (#51130) (#51348) + + This change exposes master timeout to ILM steps through global dynamic setting. + All currently implemented steps make use of this setting as well. + + Closes #44136 + +commit acf84b68cb0aa7f5579d492c4b6280e7392451ad +Author: Nhat Nguyen +Date: Thu Jan 23 08:26:27 2020 -0500 + + Do not wrap soft-deletes reader for segment stats (#51331) + + IndexWriter might not filter out fully deleted segments if retention + leases exist or the number of the retaining operations is non-zero. + SoftDeletesDirectoryReaderWrapper, however, always filters out fully + deleted segments. + + This change uses the original directory reader when calculating segment + stats instead. + + Relates #51192 + Closes #51303 + +commit 4e8ab43a3e6ecb9e2592f11ff3f845349c757a8f +Author: Armin Braun +Date: Thu Jan 23 14:29:35 2020 +0100 + + Simplify Snapshot Initialization (#51256) (#51344) + + We were loading `RepositoryData` twice during snapshot initialization, + redundantly checking if a snapshot existed already. + The first snapshot existence check is somewhat redundant because a snapshot could be + created between loading `RepositoryData` and updating the cluster state with the `INIT` + state snapshot entry. + Also, it is much safer to do the subsequent checks for index existence in the repo and + and the presence of old version snapshots once the `INIT` state entry prevents further + snapshots from being created concurrently. + While the current state of things will never lead to corruption on a concurrent snapshot + creation, it could result in a situation (though unlikely) where all the snapshot's work + is done on the data nodes, only to find out that the repository generation was off during + snapshot finalization, failing there and leaving a bunch of dead data in the repository + that won't be used in a subsequent snapshot (because the shard generation was never referenced + due to the failed snapshot finalization). + + Note: This is a step on the way to parallel repository operations by making snapshot related CS + and repo related CS more tightly correlated. + +commit 0a8d8d7ae389b2d19137fc7ea83c0d2ddb366a1b +Author: Martijn van Groningen +Date: Thu Jan 23 13:16:20 2020 +0100 + + Add Get Source API to the HLRC (#51342) + + Backport to 7.x branch of #50885. + + Relates to #47678 + + Co-authored-by: Maxim + +commit 8c8d0dbaccd4ebe7cdb9188ef2e554bada1dedc8 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 23 11:25:19 2020 +0100 + + Revert "Workaround for JDK 14 EA FileChannel.map issue (#50523)" (#51323) + + This reverts commit c7fd24ca1569a809b499caf34077599e463bb8d6. + + Now that JDK-8236582 is fixed in JDK 14 EA, we can revert the workaround. + + Relates #50523 and #50512 + +commit 157b352b4763b8df36247f815483aa238be8dda8 +Author: Nhat Nguyen +Date: Wed Jan 22 12:13:51 2020 -0500 + + Exclude nested documents in LuceneChangesSnapshot (#51279) + + LuceneChangesSnapshot can be slow if nested documents are heavily used. + Also, it estimates the number of operations to be recovered in peer + recoveries inaccurately. With this change, we prefer excluding the + nested non-root documents in a Lucene query instead. + +commit f193e527a1b66ae799177cbcf5edcb76a62374d2 +Author: Nirmal Chidambaram +Date: Wed Jan 22 11:43:15 2020 -0600 + + Short circuited to MatchNone for non-participating slice (#51207) + + In case of numSlices = numShards, use a MatchNone query + instead of boolean with a MatchNone - MUST clause + + Backport for #51207 + +commit 8a8b61735e32cf207d47b137069a5c726d986d61 +Author: Mark Vieira +Date: Wed Jan 22 13:50:48 2020 -0800 + + Add openjdk 15 to Java matrix testing rotation (#51335) + + (cherry picked from commit 9396e7574f608bb8262a9f4863c4dc637271baac) + +commit c08c282c0e4d391d22973b402a5aac0f3afea829 +Author: Mark Vieira +Date: Wed Jan 22 12:15:24 2020 -0800 + + Revert "Always test against default distribution when in a FIPS JVM (#51273)" + + This reverts commit 01694987116b462f2f5797305a8f98019d84c3e2. + This reverts commit c5a032b5947dcccf20534b8d6206fdfe5021e0de. + +commit 01694987116b462f2f5797305a8f98019d84c3e2 +Author: Mark Vieira +Date: Wed Jan 22 11:29:10 2020 -0800 + + Workaround for dependency on runtime build configuration + +commit c5a032b5947dcccf20534b8d6206fdfe5021e0de +Author: Mark Vieira +Date: Wed Jan 22 11:01:19 2020 -0800 + + Always test against default distribution when in a FIPS JVM (#51273) + + (cherry picked from commit e34d7fdaf7b511627c64a9e16805fd82f980b8c6) + +commit ec47698f7c1daf41d03dfcc636c75ec27b242a33 +Author: Lisa Cawley +Date: Wed Jan 22 11:26:10 2020 -0800 + + [DOCS] Updates categorization examples with wizard screenshots (#51133) + +commit 83647101efe28b42b028b983ef3d50ff0e7f2aa6 +Author: Zachary Tong +Date: Wed Jan 22 14:04:36 2020 -0500 + + Update Release notes for BC2 + +commit 3fb7ad0e678c48aef4f87ba73318101454eea625 +Author: Przemko Robakowski +Date: Wed Jan 22 20:03:37 2020 +0100 + + [7.x] Refactor ForEachProcessor to use iteration instead of recursion (#51104) (#51322) + + * Refactor ForEachProcessor to use iteration instead of recursion (#51104) + + * Refactor ForEachProcessor to use iteration instead of recursion + + This change makes ForEachProcessor iterative and still non-blocking. + In case of non-async processors we use single for loop and no recursion at all. + In case of async processors we continue work on either current thread or thread + started by downstream processor, whichever is slower (usually processor thread). + Everything is synchronised by single atomic variable. + + Relates #50514 + + * Update IngestCommonPlugin.java + +commit 00b0ab2bbe90bbc55efd82e626b96b6093ff79b0 +Author: Dan Pickett +Date: Wed Jan 22 12:17:33 2020 -0500 + + [DOCS] Fix typos in several REST API specs (#51197) + + Changes `effected` to `affected` in several REST API spec files. + +commit 0ac03ac5e70eef9ffd85ead727c44b5b6583d942 +Author: David Kyle +Date: Wed Jan 22 17:03:01 2020 +0000 + + [ML] Add parsers for inference configuration classes (#51300) + +commit 4590d4156ab8097e7a08943451a0a6fd6f70d40b +Author: Lisa Cawley +Date: Wed Jan 22 08:08:31 2020 -0800 + + [DOCS] Clarify interval, frequency, and bucket span in ML APIs and example (#51280) + +commit 08e9c673e5c89eea6c6f22923038e23fc65048e7 +Author: Igor Motov +Date: Wed Jan 22 05:02:48 2020 -1000 + + Fix leftover mentions of method parameter in Percentile Aggs (#51272) + + The method parameter is not used in the percentile aggs, instead + the method is determined by the presence of `hdr` or `tdigest` + objects. + + Relates to #8324 + +commit ca4b90a00149c79c2796c1e913b0e6e1f7d59c27 +Author: David Kyle +Date: Wed Jan 22 14:52:33 2020 +0000 + + [ML] Calculate results and snapshot retention using latest bucket timestamps (#51061) (#51301) + + The retention period is calculated relative to the last bucket result or snapshot + time rather than wall clock + +commit 59687a9384f154f36e8c3e6c91e9b6d780a4e71c +Author: Dimitris Athanasiou +Date: Wed Jan 22 16:51:16 2020 +0200 + + [7.x][ML] Validate classification dependent_variable cardinality is at lea… (#51232) (#51309) + + Data frame analytics classification currently only supports 2 classes for the + dependent variable. We were checking that the field's cardinality is not higher + than 2 but we should also check it is not less than that as otherwise the process + fails. + + Backport of #51232 + +commit 2a73e849d66a073dd66e55147fc3a95f5b681b79 +Author: Benjamin Trent +Date: Wed Jan 22 09:50:17 2020 -0500 + + [ML][Inference] fixing ingest IT tests (#51267) (#51311) + + Converts InferenceIngestIT into a `ESRestTestCase`. + + closes #51201 + +commit b38fdf9f949496cb597a5a62b2fa267a8fd5eb09 +Author: Zachary Tong +Date: Wed Jan 22 09:42:57 2020 -0500 + + Add release highlights for 7.6 (#51070) + + Add release highlights for 7.6 + + Co-authored-by: James Rodewig + +commit 932c63297fe61b99bcf3f5cbb005f8f0467ebf8b +Author: David Roberts +Date: Wed Jan 22 11:30:47 2020 +0000 + + [ML] Fix possible race condition when starting datafeed (#51302) + + The ID of the datafeed's associated job was being obtained + frequently by looking up the datafeed task in a map that + was being modified in other threads. This could lead to + NPEs if the datafeed stopped running at an unexpected time. + + This change reduces the number of places where a datafeed's + associated job ID is looked up to avoid the possibility of + failures when the datafeed's task is removed from the map + of running tasks during multi-step operations in other + threads. + + Fixes #51285 + +commit bfcfcdee33999ef4f9ad74aeec43b26a15bd5ad2 +Author: Przemysław Witek +Date: Wed Jan 22 12:36:24 2020 +0100 + + [7.x] Do not copy mapping from dependent variable to prediction field in regression analysis (#51227) (#51288) + +commit 1009f92b03078d61ca08bacf261d8f442eaced30 +Author: Rory Hunter +Date: Wed Jan 22 11:19:17 2020 +0000 + + Format projects under :distribution:tools (#51292) + + Backport of #51226. Opt-in the sub-projects of :distribution:tools for + automatic formatting. + +commit 421aa14972115fc0f8af5dfa5a8306955650c0e5 +Author: Andrei Dan +Date: Wed Jan 22 11:02:26 2020 +0000 + + ILM: Make UpdateSettingsStep retryable (#51235) (#51298) + + This makes the UpdateSettingsStep retryable. This step updates settings needed + during the execution of ILM actions (mark indexes as read-only, change + allocation configurations, mark indexing complete, etc) + + As the index updates are idempotent in nature (PUT requests and are applied only + if the values have changed) and the settings values are seldom user-configurable + (aside from the allocate action) the testing for this change goes along the + lines of artificially simulating a setting update failure on a particular value + update, which is followed by a successful step execution (a retry) in an + environment outside of ILM (the step executions are triggered manually). + + (cherry picked from commit 8391b0aba469f39532bfc2796b76148167dc0289) + Signed-off-by: Andrei Dan + +commit 123266714b586f18e5822f1839c34181aa046ff4 +Author: Andrei Dan +Date: Wed Jan 22 11:01:52 2020 +0000 + + ILM wait for active shards on rolled index in a separate step (#50718) (#51296) + + After we rollover the index we wait for the configured number of shards for the + rolled index to become active (based on the index.write.wait_for_active_shards + setting which might be present in a template, or otherwise in the default case, + for the primaries to become active). + This wait might be long due to disk watermarks being tripped, replicas not + being able to spring to life due to cluster nodes reconfiguration and others + and, the RolloverStep might not complete successfully due to this inherent + transient situation, albeit the rolled index having been created. + + (cherry picked from commit 457a92fb4c68c55976cc3c3e2f00a053dd2eac70) + Signed-off-by: Andrei Dan + +commit 7b4c2bfdc4b24312a47d18017675b47811dc472c +Author: Armin Braun +Date: Wed Jan 22 11:38:40 2020 +0100 + + Fix Overly Optimistic Request Deduplication (#51270) (#51291) + + On master failover we have to resent all the shard failed messages, + but the transport requests remain the same in the eyes of `equals`. + If the master failover is registered and the requests to the new master + are sent before all the callbacks have executed and the request to the + old master removed from the deduplicator then the requuests to the new + master will incorrectly fail and the snapshot get stuck. + + Closes #51253 + +commit d9ad66965cf8bfa3030407470a4c1b256d05e2b2 +Author: Armin Braun +Date: Wed Jan 22 11:38:20 2020 +0100 + + Fix Rest Tests Failing to Cleanup Rollup Jobs (#51246) (#51294) + + * Fix Rest Tests Failing to Cleanup Rollup Jobs + + If the rollup jobs index doesn't exist for some reason (like running against a 6.x cluster) + we should just assume the jobs have been cleaned up and move on. + + Closes #50819 + +commit af76ae4ab929a1b3c84a484fb92f0419013e00d3 +Author: Hendrik Muhs +Date: Wed Jan 22 11:17:59 2020 +0100 + + [Transform] Add yml test suite for testing remote clusters (CCS) (#51033) + + add a test suite for remote clusters features and add test cases for transform + +commit 86a50a24f3acd93048d623ab10b40cc6bc167572 +Author: Russ Cam +Date: Wed Jan 22 18:38:52 2020 +1000 + + [Docs] Including leading slash in range query doc example URLs (#51277) + +commit a76321437c6eea436721a95827227d181ede6c6e +Author: Ioannis Kakavas +Date: Wed Jan 22 09:56:39 2020 +0200 + + Truncate SAML Response in trace log (#51237) (#51283) + + When not truncated, a long SAML response XML document can fill max + line length and mask the actual exception message that the trace + statement is meant to inform about. + The same XML Document is also printed in full on trace level in + SamlRequestHandler#parseSamlMessage() so there is no loss of + information + +commit 4d0358cf373df67781e6937bb00149be496c254c +Author: Mark Vieira +Date: Tue Jan 21 15:16:29 2020 -0800 + + Whitelist a few more matrix parameter characters + + Signed-off-by: Mark Vieira + +commit 41c15b438d7d44494f95cb3b58e08d17309dd32f +Author: Stuart Tettemer +Date: Tue Jan 21 13:45:59 2020 -0700 + + Scripting: Add char position of script errors (#51069) (#51266) + + Add the character position of a scripting error to error responses. + + The contents of the `position` field are experimental and subject to + change. Currently, `offset` refers to the character location where the + error was encountered, `start` and `end` define a range of characters + that contain the error. + + eg. + ``` + { + "error": { + "root_cause": [ + { + "type": "script_exception", + "reason": "runtime error", + "script_stack": [ + "y = x;", + " ^---- HERE" + ], + "script": "def x = new ArrayList(); Map y = x;", + "lang": "painless", + "position": { + "offset": 33, + "start": 29, + "end": 35 + } + } + ``` + + Refs: #50993 + +commit 5d4bbdcc50feeef5198efdfc8b2213b97fd0fa73 +Author: Nhat Nguyen +Date: Tue Jan 21 12:57:30 2020 -0500 + + Use conditional doc type in testFrozenIndexAfterRestarted + +commit ca15a3f5a880c0ccee74b653508b066f017a5306 +Author: Nik Everett +Date: Tue Jan 21 12:45:52 2020 -0500 + + Add "did you mean" to unknown queries (#51177) (#51254) + + This replaces the message we return for unknown queries with the standard + one that we use for unknown fields from `ObjectParser`. This is nice + because it includes "did you mean". One day we might convert parsing + queries to using object parser, but that looks complex. This change is + much smaller and seems useful. + +commit d186e470ffc328ffea00cbeb0d387562b7e87fed +Author: Paul Sanwald +Date: Tue Jan 21 09:01:55 2020 -0500 + + version bump for 7.5.3 + +commit a9b2bc525ed7b3f50c6d7d954a88b2694e841cdc +Author: Benjamin Trent +Date: Tue Jan 21 10:29:29 2020 -0500 + + [ML] address two edge cases for categorization.GrokPatternCreator#findBestGrokMatchFromExamples (#51168) (#51255) + + There are two edge cases that can be ran into when example input is matched in a weird way. + + 1. Recursion depth could continue many many times, resulting in a HUGE runtime cost. I put a limit of 10 recursions (could be adjusted I suppose). + 2. If there are no "fixed regex bits", exploring the grok space would result in a fence-post error during runtime (with assertions turned off) + +commit 30d1587ad5196fe3b9958d8811f4dcc1a7b1473c +Author: István Zoltán Szabó +Date: Tue Jan 21 16:21:17 2020 +0100 + + [DOCS] Fixes indentation in inference processor code snippet (#51252) + +commit 6b5b26a5957c78b800fcc03f050e0a7394dda0bf +Author: Martijn van Groningen +Date: Tue Jan 21 15:37:38 2020 +0100 + + Protects against NPE: + 2> REPRODUCE WITH: ./gradlew ':x-pack:plugin:watcher:test' --tests "org.elasticsearch.xpack.watcher.history.HistoryTemplateTransformMappingsTests.testTransformFields" -Dtests.seed=26754396AB9C1A30 -Dtests.security.manager=true -Dtests.locale=lv-LV -Dtests.timezone=America/Dominica -Dcompiler.java=13 -Druntime.java=8 + 2> java.lang.NullPointerException + at __randomizedtesting.SeedInfo.seed([26754396AB9C1A30:B2A3CA27E260803B]:0) + at org.elasticsearch.xpack.watcher.history.HistoryTemplateTransformMappingsTests.lambda$testTransformFields$1(HistoryTemplateTransformMappingsTests.java:85) + at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) + at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) + at java.util.HashMap$ValueSpliterator.forEachRemaining(HashMap.java:1628) + at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482) + at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472) + at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708) + at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) + at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499) + at org.elasticsearch.xpack.watcher.history.HistoryTemplateTransformMappingsTests.lambda$testTransformFields$2(HistoryTemplateTransformMappingsTests.java:88) + at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:892) + at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:877) + at org.elasticsearch.xpack.watcher.history.HistoryTemplateTransformMappingsTests.testTransformFields(HistoryTemplateTransformMappingsTests.java:74) + +commit 788836ea3f16c144b17aa6b8205a952c0156953b +Author: Nik Everett +Date: Tue Jan 21 08:50:02 2020 -0500 + + Revert "Begin moving date_histogram to offset rounding (backport of #50873) (#50978)" (#51239) + + This reverts commit 9a3d4db840a038474dd7275cf8124f396d4eec26. It was + subtly broken in ways we didn't have tests for. + +commit 0fa7db9a9581db26222437c3dcbbf19f339264ef +Author: David Roberts +Date: Tue Jan 21 09:58:31 2020 +0000 + + [ML] Make datafeeds work with nanosecond time fields (#51180) + + Allows ML datafeeds to work with time fields that have + the "date_nanos" type _and make use of the extra precision_. + (Previously datafeeds only worked with time fields that were + exact multiples of milliseconds. So datafeeds would work + with "date_nanos" only if the extra precision over "date" was + not used.) + + Relates #49889 + +commit 0513d8dca3155bd0ef881c8c84133f913f7a2d0e +Author: Przemyslaw Gomulka +Date: Tue Jan 21 09:02:21 2020 +0100 + + Add SPI jvm option to SystemJvmOptions (#50916) + + Adding back accidentally removed jvm option that is required to enforce + start of the week = Monday in IsoCalendarDataProvider. + Adding a `feature` to yml test in order to skip running it in JDK8 + + commit that removed it 398c802 + commit that backports SystemJvmOptions c4fbda3 + relates 7.x backport of code that enforces CalendarDataProvider use #48349 + +commit 43ed244a043777c8a738c94d25c5dab372fab972 +Author: Nhat Nguyen +Date: Mon Jan 20 17:07:29 2020 -0500 + + Account soft-deletes in FrozenEngine (#51192) (#51229) + + Currently, we do not exclude soft-deleted documents when opening index + reader in the FrozenEngine. + + Backport of #51192 + +commit 79cf0894fa7ac8bf12a70329a977f425a298991f +Author: Lisa Cawley +Date: Mon Jan 20 11:08:23 2020 -0800 + + [DOCS] Adds ML PRs to release notes (#51234) + +commit 1a73d8329c97222e7fc5a0e6e7160f4435eaf9ba +Author: Adrien Grand +Date: Mon Jan 20 18:05:26 2020 +0100 + + Disable xpack/15_basic/Usage stats for mappings. + + Relates #51127 + +commit e20459c202b4cbdea3758c1706917e466e1bcb4a +Author: Jason Tedor +Date: Mon Jan 20 10:52:19 2020 -0500 + + Exclude autoscaling docs from release docs (#51190) + + Since autoscaling is currently only under development, this commit + causes the autoscaling docs to be excluded any time that release docs + are being built. + +commit 2908b7e5fcdbee099494f7af8a7192d17aa6f238 +Author: Andrei Stefan +Date: Mon Jan 20 16:40:19 2020 +0200 + + SQL: add support for passing query parameters in REST API calls (#51029) (#51222) + + * REST PreparedStatement-like query parameters are now supported in the form of an array of non-object, non-array values where ES SQL parser will try to infer the data type of the value being passed as parameter. + + (cherry picked from commit 45b8bf619aecb1c03d7bc0cf06928dcc36005a66) + +commit 543cc85b78c1c8b6ac4e5705904f17d3dd6001e6 +Author: Andrei Stefan +Date: Mon Jan 20 16:39:46 2020 +0200 + + Add trace logging for responses coming from server (#50530) (#51221) + + (cherry picked from commit 38eb485deffa175c7eb0b55a42a3e309f8a9802d) + +commit df361692206b8e7963c9186a95d1e101e47bfb9d +Author: Andrei Stefan +Date: Mon Jan 20 16:23:43 2020 +0200 + + SQL: change the way unsupported data types fields are handled (#50823) (#51220) + + The hierarchy of fields/sub-fields under a field that is of an + unsupported data type will be marked as unsupported as well. Until this + change, the behavior was to set the unsupported data type field's + hierarchy as empty. + + Example, considering the following hierarchy of fields/sub-fields + a -> b -> c -> d, if b would be of type "foo", then b, c and d will + be marked as unsupported. + + (cherry picked from commit 7adb286c4c485b9e781f88b0a2f98cab9ec5b7e2) + +commit 51134d9738681c3d35455a9c8818e45f766a5b4e +Author: Hendrik Muhs +Date: Mon Jan 20 13:52:19 2020 +0100 + + check custom meta data to avoid NPE (#51163) + + check custom meta data to avoid NPE, fixes a problem introduced in #51072 + + fixes #51153 + +commit dc02458dd630e628af1b0ec72bd6c20db2d70dfe +Author: zacharymorn +Date: Mon Jan 20 02:53:38 2020 -0800 + + Exclude unmapped fields from query max_clause limit (#49523) + + Take into account of number of unmapped fields when calculating against limit. + Closes #49002 + +commit c5f1a901599636dbb0ec51b593200156e286f826 +Author: Armin Braun +Date: Mon Jan 20 12:18:16 2020 +0100 + + Add CoolDown Period to S3 Repository (#51074) (#51213) + + Add cool down period after snapshot finalization and delete to prevent eventually consistent AWS S3 from corrupting shard level metadata as long as the repository is using the old format metadata on the shard level. + +commit 424b4ed4eab54f046c23623d2ceb74adc488755b +Author: István Zoltán Szabó +Date: Mon Jan 20 11:11:57 2020 +0100 + + [DOCS] Expands the documentation of Node Query Cache (#51105) + + Co-authored-by: debadair + +commit 4b31ad1c0c870efe4c1e87c1d54bd3a5f61af54e +Author: Jess <13388033+liebeslied@users.noreply.github.com> +Date: Mon Jan 20 10:29:36 2020 +0100 + + [Docs] Small edits to Ranking Evaluation API docs (#51116) + + Small updates to grammar, syntax, and unclear wordings. + +commit a0ca82422cadef4c5764c68bc70304f7ce7fcc5c +Author: Tim Vernum +Date: Mon Jan 20 20:19:29 2020 +1100 + + Mute TimeSeriesLifecycleActionsIT.waitForSnapshot (#51208) + + This test was recently un-muted, but is still failing + + Relates: #50781 + Backport of: #51203 + +commit e40580c24df38b2db0375e7681b0965bbee09195 +Author: István Zoltán Szabó +Date: Mon Jan 20 09:42:04 2020 +0100 + + [DOCS] Removes CCS limitation item from Transforms limitations. (#51151) + +commit 815cceff9c98aeb942de13c9985bf9e8c2b8b9e0 +Author: Tim Vernum +Date: Mon Jan 20 17:38:04 2020 +1100 + + Bump docs test suite timeout to 40min (#51204) + + The docs test suite is still timing out on CI at 35 minutes, so + pushing it to 40 minutes while we determine the cause of the slowdown. + + Relates: #49753 + Backport of: #51200 + +commit 694b8ab95dd298bebd9c1021efb10653cd87afae +Author: Armin Braun +Date: Sat Jan 18 14:12:55 2020 +0100 + + Fix CorruptedBlobStoreRepository Test (#51128) (#51186) + + The tests, when creating broken serialized blobs could randomly create + a sequence of bytes that is partially readable by the deserializer and then + not throw `IOException` but instead `ElasticsearchParseException`. + We should just handle these unexpected exceptions downstream properly and pass them + wrapped as `RepositoryException` to the listener to fix the test and keep the API consistent. + +commit e6f778474e8f6d052ba7198fc49c03f386a5b32b +Author: Rory Hunter +Date: Sat Jan 18 10:39:49 2020 +0000 + + Follow symlinks in Docker entrypoint (#51101) + + Backport of #50927. + + Closes #49653. When using _FILE environment variables to supply values + to Elasticsearch, following symlinks when checking that file permissions + are secure. + +commit 977b53ab915768024cb85dc1e887c4f86731d647 +Author: Nik Everett +Date: Fri Jan 17 16:53:13 2020 -0500 + + Fix flaky usage tracking test (#51169) (#51179) + + We added tracking of index feature usage in #51031 but due to some copy + and paste errors the test fails on some seeds. This fixes those errors. + +commit 9ce4d2b9018ecdb5a269a4d3adff23abd04593aa +Author: Jason Tedor +Date: Fri Jan 17 14:54:53 2020 -0500 + + Initial autoscaling commit (#51161) + + This commit merely adds the skeleton for the autoscaling project, adding + the basics to include the autoscaling module in the default + distribution, opt-in to code formatting, and a placeholder for the docs. + +commit 731c96b5074c6dbde61a454605b8fae83baccbad +Author: Lee Hinman +Date: Fri Jan 17 13:26:40 2020 -0700 + + [7.x] Use separate policies for tests in SnapshotLifecycleRest… (#51181) + + These policies store statistics, but since stats updating is asynchronous, it's + possible for the update from one test to bleed into a separate one. This change + switches the tests to use separate policy ids so that their stats are tracked + independently. It also relaxes the checking constraint in one of the tests. + + Hopefully this: + Resolves #48531 + Resolves #48017 + +commit 83c92cf7ebee262876fbd8e409f01420194d7c57 +Author: István Zoltán Szabó +Date: Fri Jan 17 18:52:57 2020 +0100 + + [DOCS] Adds text about data types to the categorization docs (#51145) + +commit ccf3e443b555646323411938f3ace03a6ed97db3 +Author: Jay Modi +Date: Fri Jan 17 10:56:22 2020 -0700 + + Improve file filter for insecure repo tests (#51121) (#51170) + + Tests in BuildPluginIT copy the workspace but exclude the build + directories based on whether the directory string representation + includes `/build/` or not. This check is problematic if the directory + of the project has a parent directory also named `build`. The change in + this commit checks to see if the path relative to the project directory + has any path parts equal to `build`. + +commit fee1a9528cac4a7b31fff8204f0985b3f5b1237c +Author: lcawl +Date: Fri Jan 17 09:49:42 2020 -0800 + + [DOCS] Removes duplicate title + +commit 107989df3ee68f4e07bc6cab89e32b0dcec8ad77 +Author: Jay Modi +Date: Fri Jan 17 10:09:01 2020 -0700 + + Introduce hidden indices (#51164) + + This change introduces a new feature for indices so that they can be + hidden from wildcard expansion. The feature is referred to as hidden + indices. An index can be marked hidden through the use of an index + setting, `index.hidden`, at creation time. One primary use case for + this feature is to have a construct that fits indices that are created + by the stack that contain data used for display to the user and/or + intended for querying by the user. The desire to keep them hidden is + to avoid confusing users when searching all of the data they have + indexed and getting results returned from indices created by the + system. + + Hidden indices have the following properties: + * API calls for all indices (empty indices array, _all, or *) will not + return hidden indices by default. + * Wildcard expansion will not return hidden indices by default unless + the wildcard pattern begins with a `.`. This behavior is similar to + shell expansion of wildcards. + * REST API calls can enable the expansion of wildcards to hidden + indices with the `expand_wildcards` parameter. To expand wildcards + to hidden indices, use the value `hidden` in conjunction with `open` + and/or `closed`. + * Creation of a hidden index will ignore global index templates. A + global index template is one with a match-all pattern. + * Index templates can make an index hidden, with the exception of a + global index template. + * Accessing a hidden index directly requires no additional parameters. + + Backport of #50452 + +commit 96e8f674257de7f4a3131809a6efe7cb6414a1ff +Author: Jay Modi +Date: Fri Jan 17 10:00:33 2020 -0700 + + Upgrade to the latest OWASP HTML sanitizer (#50765) (#51166) + + This commit upgrades the OWASP HTML sanitizer used by watcher to the + latest version and also upgrades guava, which it depends on. The guava + upgrade also requires the addition of a new dependency that guava + itself requires as of version 27.0. The sanitizer's behavior has changed to + re-write these templated values with a comment that results in this output + `{{ctx.metadata.name}}`. This would be an issue if we attempted to + sanitize the template, but the code that uses the sanitizer runs the rendered + string through the sanitizer, which means that the templated values have + been replaced already. + + Relates #50395 + +commit 5299664ae345a6d5244343964b4c222e54bb5b44 +Author: Nik Everett +Date: Fri Jan 17 12:00:03 2020 -0500 + + "did you mean" for ObjectParser with top named (#51018) (#51165) + + When you declare an ObjectParser with top level named objects like we do + with `significant_terms` we didn't support "did you mean". This fixes + that. + + Relates #50938 + +commit 4fc865e579b9d3b56972a869d8cedf576cdc1700 +Author: Ioannis Kakavas +Date: Fri Jan 17 18:56:02 2020 +0200 + + Don't fallback to anonymous for tokens/apikeys (#51042) (#51159) + + This commit changes our behavior so that when we receive a + request with an invalid/expired/wrong access token or API Key + we do not fallback to authenticating as the anonymous user even if + anonymous access is enabled for Elasticsearch. + +commit 295665b1ea941e65b6d2f2b40797e618c51b2229 +Author: David Roberts +Date: Fri Jan 17 16:25:06 2020 +0000 + + [ML] Add audit warning for 1000 categories found early in job (#51146) + + If 1000 different category definitions are created for a job in + the first 100 buckets it processes then an audit warning will now + be created. (This will cause a yellow warning triangle in the + ML UI's jobs list.) + + Such a large number of categories suggests that the field that + categorization is working on is not well suited to the ML + categorization functionality. + +commit da73c9104e41747b4547155dc10b410a72132f51 +Author: Przemysław Witek +Date: Fri Jan 17 14:58:58 2020 +0100 + + [ML] Fix tests randomly failing on CI (#51142) (#51150) + +commit b70ebdeb96bd10cb7bdd3f47793e4005777f1113 +Author: Dimitris Athanasiou +Date: Fri Jan 17 14:02:59 2020 +0200 + + [7.x][ML] DF Analytics _explain API should skip object fields (#51115) (#51147) + + Object fields cannot be used as features. At the moment _explain + API includes them and even worse it allows it does not error when + an object field is excluded. This creates the expectation to the + user that all children fields will also be excluded while it's not + the case. + + This commit omits object fields from the _explain API and also + adds an error if an object field is included or excluded. + + Backport of #51115 + +commit 82827442073b6ba53663cbde3ae0572efda32fe3 +Author: Hendrik Muhs +Date: Fri Jan 17 12:57:05 2020 +0100 + + increase timeout of DocsClientYamlTestSuiteIT + + increase timeout of DocsClientYamlTestSuiteIT to 35 minutes, temporary solution for issue #49753 + +commit b1a526d5e9dbe8ea39754c147a701ef980028439 +Author: Przemysław Witek +Date: Fri Jan 17 08:12:54 2020 +0100 + + [7.x] [ML] Update DFA progress document in the index the document belongs to (#51111) (#51117) + +commit 13343b15c908d29560c9bb549311e6ea2ce84588 +Author: Hendrik Muhs +Date: Fri Jan 17 07:40:02 2020 +0100 + + [Transform] Improve force stop robustness in case of an error (#51072) + + If a transform config got lost (e.g. because the internal index disappeared) tasks could not be + stopped using transform API. This change makes it possible to stop transforms without a config, + meaning to remove the background task. In order to do so force must be set to true. + +commit fac509836aae779bd7aab3cc64c55d785714c498 +Author: Mark Vieira +Date: Thu Jan 16 11:08:07 2020 -0800 + + Upgrade to Gradle 6.1 (#50453) + + (cherry picked from commit bdd7bda47b4aa14a14273f7301d3615f1cc4bba0) + +commit c794803e3d15cff8eb6cb01a1168a150c65b7330 +Author: Armin Braun +Date: Thu Jan 16 21:52:57 2020 +0100 + + Fix X-Pack Indices Breaking Repository BwC Tests (#51120) (#51125) + + If some internal `.watcher` or so index gets created during these tests + then the shard counts on snapshot restores and creates won't match up with expectations. + Fixed by only creating the snapshot for the test index + + Closes #50819 + +commit e51b209dd35f4153bdc9be9f4cd2c083b14e6c28 +Author: Armin Braun +Date: Thu Jan 16 21:08:35 2020 +0100 + + Fix Infinite Retry Loop in loading RepositoryData (#50987) (#51093) + + * Fix Infinite Retry Loop in loading RepositoryData + + We were running into an infinite loop when trying to load corrupted (or otherwise un-loadable) + repository data for a repo that uses best effort consistency (e.g. that was just freshly mounted + as done in the test) because we kepy resetting to `-1` on `IOException`, listing and finding the broken + generation `N` and then interpreted the subsequent reset to `-1` as a concurrent change to the repository. + +commit f6c89b4599032b31e3bb889e96c1aedcb959e422 +Author: Nik Everett +Date: Thu Jan 16 14:49:12 2020 -0500 + + Move test of custom sig heuristic to plugin (#50891) (#51067) + + This moves the testing of custom significance heuristic plugins from an + `ESIntegTestCase` to an example plugin. This is *much* more "real" and + can be used as an example for anyone that needs to actually build such a + plugin. The old test had testing concerns and the example all jumbled + together. + +commit 2353fe47fcaefc9cf9232c42c1bb94f58551a655 +Author: James Rodewig +Date: Thu Jan 16 14:34:39 2020 -0500 + + [DOCS] Adds placeholder for 7.5.2 release notes (#51124) + + Co-Authored-By: Lisa Cawley + +commit b6bf64b96927c93d41aaa99f9200143f50a0d58a +Author: James Rodewig +Date: Thu Jan 16 13:18:36 2020 -0500 + + [DOCS] Collapse node stats response sections (#51063) + + elastic/docs#1687 added support for the `[%collapsible]` Asciidoc + attribute, which creates collapsible sections in the HTML output. + + This PR makes two related changes to the nodes stats API documentation: + + * Makes the response parameter sections collapsible. This allows users + to more easily navigate the page without long walls of text. + + * Reorders the response parameter sections to match the default order + returned by the API. + + Relates to #47524. + +commit 7ef906fde81b59c7399a6cb7ccaa3a3456e966b5 +Author: James Rodewig +Date: Thu Jan 16 13:11:42 2020 -0500 + + [DOCS] Add tutorials section to analysis topic (#50809) + + Adds a 'Configure text analysis' page to house tutorial content for the + analysis topic. + + Also relocates the following pages as children as this new page: + + * 'Test an analyzer' + * 'Configuring built-in analyzers' + * 'Create a custom analyzer' + + I plan to add a tutorial for specifying index-time and search-time + analyzers to this section as part of a future PR. + +commit ef26763ca9b620d7afbe76874adb1f94f80f9b55 +Author: James Rodewig +Date: Thu Jan 16 13:00:04 2020 -0500 + + [DOCS] Add concepts section to analysis topic (#50801) + + This helps the topic better match the structure of + our machine learning docs, e.g. + https://www.elastic.co/guide/en/machine-learning/7.5/ml-concepts.html + + This PR only includes the 'Anatomy of an analyzer' page as a 'Concepts' + child page, but I plan to add other concepts, such as 'Index time vs. + search time', with later PRs. + +commit 1edaf2b1019c98a42116bd0e0b011f3ebfcd63fc +Author: James Rodewig +Date: Thu Jan 16 12:27:54 2020 -0500 + + [DOCS] Retitle analysis reference pages (#51071) + + * Changes titles to sentence case. + + * Appends pages with 'reference' to differentiate their content from + conceptual overviews. + + * Moves the 'Normalizers' page to end of the Analysis topic pages. + +commit d590150ca25263d6a8773399ea0ad2dbc5668929 +Author: James Rodewig +Date: Thu Jan 16 10:59:18 2020 -0500 + + [DOCS] Fix indent issue in similarity snippet (#51107) + + Updates snippet to consistently use 2-space indentation. The snippet + previously used a mix of tab/5-space and 2-space indents. + + Co-authored-by: Peter Johnson + + Co-authored-by: Peter Johnson + +commit e2ca93bad309286f6864dc7cf1e1f2264f2f73eb +Author: Zachary Tong +Date: Thu Jan 16 10:23:24 2020 -0500 + + Mute GeoGridAggregatorTestCase#testBounds() + + Tracking issue: https://github.com/elastic/elasticsearch/issues/51103 + +commit d0554fd317c9f8101bca0644519fb8f59f9fc397 +Author: Ioannis Kakavas +Date: Thu Jan 16 17:00:17 2020 +0200 + + Fail gracefully on invalid token strings (#51014) (#51096) + + When we receive a request with an Authorization header that contains + a Bearer token that is not generated by us or that is malformed in + some way, attempting to decode it as one of our own might cause a + number of exceptions that are not IOExceptions. This commit ensures + that we catch and log these too and call onResponse with `null, so + that we can return 401 instead of 500. + + Resolves: #50497 + +commit 584cb0d926cb534890010937fde787d25daa42cb +Author: Florian Kelbert +Date: Thu Jan 16 15:34:25 2020 +0100 + + [DOCS] Correctly read total hits inside watcher config (#50614) + + With elastic/elasticsearch#35848, users can now retrieve total hits as an integer when the `rest_total_hits_as_int` query parameter is `true`. This is the default value. + + This updates several snippet examples in the Watcher docs that used a workaround to get a total hits integer. + +commit 1211772f6ad0f237067912b48a2b63e0d1bb349a +Author: James Rodewig +Date: Thu Jan 16 09:13:31 2020 -0500 + + [DOCS] Use same index in Cluster Allocation Explain docs (#50936) + + Updates several example snippets in the Cluster Allocation Explain API + docs to consistently use the `my_index` index. + + Previously, the snippets switches from `my_index` to `idx`, which could + confuse users. + + Co-authored-by: Emmanuel DEMEY + + Co-authored-by: Emmanuel DEMEY + +commit fb65ef3f2d6833f86e69de29ac4d39b29e253aec +Author: Bogdan Pintea +Date: Thu Jan 16 14:32:34 2020 +0100 + + SQL: Extend the optimisations for equalities (#50792) (#51098) + + * Extend the optimizations for equalities + + This commit supplements the optimisations of equalities in conjunctions + and disjunctions: + * for conjunctions, the existing optimizations with ranges are extended + with not-equalities and inequalities; these lead to a fast resolution, + the conjunction either being evaluate to a FALSE, or the non-equality + conditions being dropped as superfluous; + * optimisations for disjunctions are added to be applied against ranges, + inequalities and not-equalities; these lead to disjunction either + becoming TRUE or the equality being dropped, either as superfluous or + merged into a range/inequality. + + * Adress review notes + + * Fix the bug around wrongly optimizing 'a=2 OR a!=?', which only yields + TRUE for same values in equality and inequality. + * Var renamings, code style adjustments, comments corrections. + + * Address further review comments. Extend optim. + + - fix a few code comments; + - extend the Equals OR NotEquals optimitsation (a=2 OR a!=5 -> a!=5); + - extend the Equals OR Range optimisation on limits equality (a=2 OR + 2<=a<5 -> 2<=a<5); + - in case an equality is being removed in a conjunction, the rest of + possible optimisations to test is now skipped. + + * rename one var for better legiblity + + - s/rmEqual/removeEquals + + (cherry picked from commit 62e7c6a010f10cd7893ee5c99bad8b8d2a693436) + +commit fda25ed04a5510314a9c6a830475d27c32fa59e0 +Author: Marios Trivyzas +Date: Thu Jan 16 13:58:02 2020 +0100 + + Fix caching for PreConfiguredTokenFilter (#50912) (#51091) + + The PreConfiguredTokenFilter#singletonWithVersion uses the version + internally for the token filter factories but it registers only one + instance in the cache and not one instance per version. This can lead + to exceptions like the one described in #50734 since the singleton is + created and cached using the version created of the first index + that is processed. + + Remove the singletonWithVersion() methods and use the + elasticsearchVersion() methods instead. + + Fixes: #50734 + (cherry picked from commit 24e1858) + +commit 32ec934b154eecbe63952a212609c18c3c2086dd +Author: Tom Veasey +Date: Thu Jan 16 12:23:15 2020 +0000 + + [7.x][ML] Assert top classes are ordered by score (#51028) + + Backport #51003. + +commit b345c7ff3107773adada532aaddbbe4f286ad480 +Author: Ted Timmons +Date: Fri Jan 17 00:09:54 2020 +1300 + + [Docs] Fix short alias for 'unassigned.for' (#51059) + + The short alias for `unassigned.for` is `uf`, not 'ua'. + +commit ff0a45f8820480f49277f83fc5556837fe0e8c25 +Author: markharwood +Date: Thu Jan 16 10:41:43 2020 +0000 + + Fix NPE in PinnedQuery call to DisjunctionMaxScorer. (#51047) (#51064) + + Fix NPE in PinnedQuery call to DisjunctionMaxScorer. (#51047) + Added test and fix that tests for score type. + Closes #51034 + +commit 1d391f711381db64038443d041f643b8ce51ac21 +Author: PND +Date: Thu Jan 16 19:32:55 2020 +0900 + + [Docs] Fix example output of edge n-gram token filter. (#51085) + +commit 80d925e22516fdbc81d2a01e87b3fae652a06062 +Author: Rory Hunter +Date: Thu Jan 16 10:26:27 2020 +0000 + + Auto-format buildSrc (#51043) + + Backport / reimplementation of #50786 on 7.x. + + Opt-in `buildSrc` for automatic formatting. This required a config tweak + in order to pick up all the Java sources, and as a result more files are + now found in the Enrich plugin, that were previously missed. + + I also moved the 2 Java files in `buildSrc/src/main/groovy` into the Java + directory, which required some follow-up changes. + +commit 02dfd71efa80bb992b4c04231c9c841b7b2be73f +Author: Martijn van Groningen +Date: Thu Jan 16 10:50:47 2020 +0100 + + Backport: Add pipeline name to ingest metadata (#51050) + + Backport: #50467 + + This commit adds the name of the current pipeline to ingest metadata. + This pipeline name is accessible under the following key: '_ingest.pipeline'. + + Example usage in pipeline: + PUT /_ingest/pipeline/2 + { + "processors": [ + { + "set": { + "field": "pipeline_name", + "value": "{{_ingest.pipeline}}" + } + } + ] + } + + Closes #42106 + +commit 45d7bdcfd7847d47eb5be178c887d9d5dc09bd25 +Author: Adrien Grand +Date: Thu Jan 16 09:56:41 2020 +0100 + + Add analysis components and mapping types to the usage API. (#51062) + + Knowing about used analysis components and mapping types would be incredibly + useful in order to know which ones may be deprecated or should get more love. + + Some field types also act as a proxy to know about feature usage of some APIs + like the `percolator` or `completion` fields types for percolation and the + completion suggester, respectively. + +commit ac6602a156d26a56c500f62594fcfd2e7e7ba2b4 +Author: Tim Vernum +Date: Thu Jan 16 17:01:58 2020 +1100 + + Fix windows newline issue in test (#51082) + + Fixes HttpCertificateCommandTests.testTextFileSubstitutions on Windows + + Backport of: #51030 + +commit c1a6d5d9ffead45efd3238432d2cca3add54fa9f +Author: Yang Wang +Date: Thu Jan 16 14:47:21 2020 +1100 + + Encrypt generated key with AES (#51019) (#51076) + + Replace DES with AES to align with modern encryption standards + Backport also fixs Files.readString API that is not available in Java 8 + + Resolves: #50843 + +commit dd419f0ad099ff222c49925bb4a56b57f55e4b08 +Author: Oleksandr Stasyk +Date: Wed Jan 15 21:07:21 2020 +0000 + + [DOCS] Add Elixir Bulk Processor to community clients (#50630) + +commit 18d1b7aecdca158bfc65c547d6f5cf20c31fe0e4 +Author: James Rodewig +Date: Wed Jan 15 15:20:02 2020 -0500 + + [DOCS] Add note to community clients page (#51065) + + Adds a note to the 'Community Contributed Clients' page explicitly + stating that Elastic does not support or endorse the clients. + +commit a62c9e4e698c45b02d6b19985b724a270892bb6e +Author: Zachary Tong +Date: Wed Jan 15 15:16:48 2020 -0500 + + Fix 7.6 release notes file name + +commit 8f48c8d312bdae3b908244c4ffef6f0f7241ef57 +Author: Zachary Tong +Date: Wed Jan 15 14:06:59 2020 -0500 + + Add 7.6.0 release notes + +commit 2d1c28a45d8db0a676a1436b5610f0fb671f3b68 +Author: Lee Hinman +Date: Wed Jan 15 11:36:34 2020 -0700 + + [7.x] Fix AllocateRoutedStepTests reusing keys for random valu… (#51058) + + In these tests there was a very small chance that keys could collide, + which causes test failures. + + Resolves #49307 + +commit e395cf3419fb5728078462db84783822a42be135 +Author: Lee Hinman +Date: Wed Jan 15 11:21:18 2020 -0700 + + Guard against null settings in CCRIndexLifecycleIT (#51008) (#51054) + + It's possible that the index could return no settings and thus throw a + `NullPointerException`. + + I wasn't able to reproduce the original issue, but this should guard + against in the future. + + Resolves #50646 + + Co-authored-by: Elastic Machine + +commit d291f189a8da4843715de25ccaa1178690987cc8 +Author: Christoph Büscher +Date: Wed Jan 15 18:09:37 2020 +0100 + + Fix hardcoded version replacement in put-dfanalytics.asciidoc #51053 + + The version replacement for the code snippet should replace 7.6 with the current version, + but doesn't match because of a missing whitespace. + + Closes #51052 + +commit 65af0a0f0ab7d6b8550573bb20cea9d15d072746 +Author: taku333 <19281989thezuka@gmail.com> +Date: Thu Jan 16 01:52:22 2020 +0900 + + [DOCS] Add 7.5.1 link to release notes overview (#51022) + +commit ad60f0015e0083762529a9d44612a534d3fd0f93 +Author: Lee Hinman +Date: Wed Jan 15 09:47:41 2020 -0700 + + Address failures in SnapshotLifecycleRestIT.testFullPolicySnapshot (#51013) + + This test failed a couple of different ways, related to timing, as well + as concurrent snapshots, and also naming. + + This commit splits the giant `assertBusy` into separate parts so that we don't + perform ~5 different requests and tests in the same loop. It also gives each + test a unique repository so that no other test can accidentally re-use + snapshots. + + Resolves #50358 (hopefully!) + +commit b14a949fa929c79202d991795a7f1d9194205c0b +Author: Lee Hinman +Date: Wed Jan 15 09:46:34 2020 -0700 + + Add blurb about ILM-injected unfollow action (#51009) + + These injected actions are harmless and safe to ignore for non-CCR indices. + + Resolves #50548 + +commit 5ea18cb4af4572ec72b928f7c1750470581f0597 +Author: Robin Clarke +Date: Wed Jan 15 16:15:53 2020 +0100 + + [Docs] Fix sub-heading in start-stop-ilm.asciidoc (#51045) + + Removed superfluous `=`. + +commit 9eab87062c51f928885b9e94f01d7a47e136e35b +Author: Zachary Tong +Date: Wed Jan 15 10:12:14 2020 -0500 + + Bump version to 7.7.0 + +commit dc47b380c8fda6bce358254855e32c662956ad4f +Author: Yannick Welsch +Date: Wed Jan 15 15:50:08 2020 +0100 + + Block too many concurrent mapping updates (#51038) + + Ensures that there are not too many concurrent dynamic mapping updates going out from the + data nodes to the master. + + Closes #50670 + +commit b4a631277a5464b2504013bf662799e03f139bf3 +Author: Przemysław Witek +Date: Wed Jan 15 15:53:42 2020 +0100 + + Add missing docs for new evaluation metrics (#50967) (#51041) + +commit dd09dc7af6296cb4c698ebc02afafb70442d242b +Author: Armin Braun +Date: Wed Jan 15 15:19:08 2020 +0100 + + Fix MultiVersionRepository BwC Tests (#51006) + + * Fix MultiVersionRepository BwC Tests + + The HLRC doesn't like what its getting back from some older 6.x versions for the restore status + so I moved that request to the low level client. + Closes #50819 + +commit 2f069d8f3fdfe92c143e562445498c514e75643c +Author: Rory Hunter +Date: Wed Jan 15 13:17:37 2020 +0000 + + Tweak formatter config for long generic lines (#51027) + + Backport of #50909. The current formatting config allows some long + generic declarations to break the 140 character limit. Tweak the config + to wrap such lines. + +commit 0257de8c26fdd8fcaac2c78f7db67fe74efd5a18 +Author: Alan Woodward +Date: Wed Jan 15 11:59:27 2020 +0000 + + Emit warnings when index templates have multiple mappings (#50982) + + Index templates created in the 5x line can still be present in the cluster state + through multiple upgrades, and may have more than one mapping defined. 8x + will stop supporting templates with multiple mappings, and we should emit + deprecation warnings in 7x clusters to give users a chance to update their + templates before upgrading. + +commit 1536c3e6228923824a0926e184c52609407c5e7b +Author: David Roberts +Date: Wed Jan 15 08:55:46 2020 +0000 + + [TEST] Increase ML distributed test job open timeout (#50998) + + There have been occasional failures, presumably due to + too many tests running in parallel, caused by jobs taking + around 15 seconds to open. (You can see the job open + successfully during the cleanup phase shortly after the + failure of the test in these cases.) This change increases + the wait time from 10 seconds to 20 seconds to reduce the + risk of this happening. + +commit e76c3d4d3260deb2ea301dfdf7f5ffc5aa026195 +Author: Martijn van Groningen +Date: Wed Jan 15 08:50:50 2020 +0100 + + Tidy up enrich processors: (#50957) + + * Fix generics usages. + * Sealed match processor class. + +commit 5b6fa79fd8f214448dd55a468e16c4561df377c0 +Author: Tomas Della Vedova +Date: Wed Jan 15 08:50:59 2020 +0100 + + [ML] Removed key value from the catch regex test (#50977) (#51021) + +commit b570f417c298aa7995120272e3196ec9ac4986ce +Author: István Zoltán Szabó +Date: Wed Jan 15 08:45:03 2020 +0100 + + [DOCS] Describes the relationship of the time-related settings in anomaly detection docs (#50959) + + Co-Authored-By: David Roberts + +commit e41c0b1224001a4dc8014279056fa438f43ab1cc +Author: Tim Vernum +Date: Wed Jan 15 11:07:19 2020 +1100 + + Deprecating kibana_user and kibana_dashboard_only_user roles (#50963) + + This change adds a new `kibana_admin` role, and deprecates + the old `kibana_user` and`kibana_dashboard_only_user`roles. + + The deprecation is implemented via a new reserved metadata + attribute, which can be consumed from the API and also triggers + deprecation logging when used (by a user authenticating to + Elasticsearch). + + Some docs have been updated to avoid references to these + deprecated roles. + + Backport of: #46456 + + Co-authored-by: Larry Gregory + +commit f207e5bde96d743fdbf86d94289087c9709a4061 +Author: Gordon Brown +Date: Tue Jan 14 16:18:28 2020 -0700 + + Add generated benchmark files to gitignore (#51000) + + When building IntelliJ generates several source files related to the + benchmarks. This commit adds that path to the gitignore so these don't + get accidentally committed. + +commit fc5fde795024d8ac90ded2ca53d888d9325acf26 +Author: Nik Everett +Date: Tue Jan 14 17:53:41 2020 -0500 + + Add "did you mean" to ObjectParser (#50938) (#50985) + + Check it out: + ``` + $ curl -u elastic:password -HContent-Type:application/json -XPOST localhost:9200/test/_update/foo?pretty -d'{ + "dac": {} + }' + + { + "error" : { + "root_cause" : [ + { + "type" : "x_content_parse_exception", + "reason" : "[2:3] [UpdateRequest] unknown field [dac] did you mean [doc]?" + } + ], + "type" : "x_content_parse_exception", + "reason" : "[2:3] [UpdateRequest] unknown field [dac] did you mean [doc]?" + }, + "status" : 400 + } + ``` + + The tricky thing about implementing this is that x-content doesn't + depend on Lucene. So this works by creating an extension point for the + error message using SPI. Elasticsearch's server module provides the + "spell checking" implementation. + s + +commit 4bdab0e985acf67aaedaee8364f76788c62dda4d +Author: Ryan Ernst +Date: Tue Jan 14 14:39:30 2020 -0800 + + Fix windows chown to work with single file (#51004) + + The chown utility for packaging tests works on windows when the given + path is a directory, but would fail if the path was a single file. This + commit fixes it to handle both cases. + + relates #50825 + +commit 4b0581f182467b6f40945e7f385a3a0a7ba80acc +Author: Yannick Welsch +Date: Tue Jan 14 18:33:53 2020 +0100 + + Remove custom metadata tool (#50813) + + Adds a command-line tool to remove broken custom metadata from the cluster state. + + Relates to #48701 + +commit a290762df1ec18feb4399b0451f76a7e9d43a11d +Author: James Rodewig +Date: Tue Jan 14 16:48:35 2020 -0500 + + [DOCS] Document `breakers`, `script`, and `discovery` node stats (#50509) + + Documents the `breakers`, `script`, and `discovery` parameters returned + by the `_nodes/stats` API. + +commit a8aca6b2a04eb20f34792f653e982e0ffc7db5b4 +Author: Nik Everett +Date: Tue Jan 14 16:50:52 2020 -0500 + + Switch AggregationSpec to ContextParser (#50871) (#50980) + + We seem to have settled on the `ContextParser` interface for parsing + stuff, mostly because `ObjectParser` implements it. We don't really + *need* the old `Aggregator.Parser` interface any more because it + duplicates `ContextParser` but with the arguments reversed. + + This adds support to `AggregationSpec` to declare aggregation parsers + using `ContextParser`. This should integrate cleanly with + `ObjectParser`. It doesn't drop support for `Aggregator.Parser` or + change the plugin intrface at all so it *should* be safe to backport to + 7.x. And we can remove `Aggregator.Parser` in a follow up which is only + targeted to 8.0. + +commit 9a3d4db840a038474dd7275cf8124f396d4eec26 +Author: Nik Everett +Date: Tue Jan 14 16:50:27 2020 -0500 + + Begin moving date_histogram to offset rounding (backport of #50873) (#50978) + + We added a new rounding in #50609 that handles offsets to the start and + end of the rounding so that we could support `offset` in the `composite` + aggregation. This starts moving `date_histogram` to that new offset. + +commit d5623c8f098dda86759c82279f46c3b9cbd71b52 +Author: Jason Tedor +Date: Tue Jan 14 16:23:58 2020 -0500 + + Report progress of multiple plugin installs (#51001) + + When installing multiple plugins at once, this commit changes the + behavior to report installed plugins as we go. In the case of failure, + we emit a message that we are rolling back any plugins that were + installed successfully, and also that they were successfully rolled + back. In the case a plugin is not successfully rolled back, we report + this clearly too, alerting the user that there might still be state on + disk they would have to clean up. + +commit 2f13751badb4b08028f6ad2bb05edef35b2c8838 +Author: Christoph Büscher +Date: Tue Jan 14 21:42:34 2020 +0100 + + Deprecate and remove camel-case nGram and edgeNGram tokenizers (#50862) (#50991) + + We deprecated and removed the camel-case versions of the nGram and edgeNGram + filters a while ago and we should do the same with the nGram and edgeNGram tokenizers. + This PR deprecates the use of these names in favour of ngram and edge_ngram in + 7. Usage will be disallowed on new indices starting with 8 then. + +commit 6848dee84b30ec07efde193b7e496e1f58c6d471 +Author: lcawl +Date: Tue Jan 14 11:56:08 2020 -0800 + + [DOCS] Fixes typo in keystore command + +commit 9ee2e111813f2523b56bf45ae83fb4a588afc6d8 +Author: Tal Levy +Date: Tue Jan 14 11:18:46 2020 -0800 + + [7.x] Adds support for geo-bounds filtering in geogrid aggregations (#50996) + + * Adds support for geo-bounds filtering in geogrid aggregations (#50002) + + It is fairly common to filter the geo point candidates in + geohash_grid and geotile_grid aggregations according to some + viewable bounding box. This change introduces the option of + specifying this filter directly in the tiling aggregation. + + This is even more relevant to `geo_shape` where the bounds will restrict + the shape to be within the bounds + + this optional `bounds` parameter is parsed in an equivalent fashion to + the bounds specified in the geo_bounding_box query. + +commit 2de2a3634e1d78e3267648339e06170fd375989f +Author: Mark Vieira +Date: Tue Jan 14 09:53:46 2020 -0800 + + Add matrix job params as build scan tag + + Signed-off-by: Mark Vieira + +commit 72c270946fe1137b98aadb1a304b505f18673eed +Author: Benjamin Trent +Date: Tue Jan 14 12:40:25 2020 -0500 + + [ML][Inference] Adding classification_weights to ensemble models (#50874) (#50994) + + * [ML][Inference] Adding classification_weights to ensemble models + + classification_weights are a way to allow models to + prefer specific classification results over others + this might be advantageous if classification value + probabilities are a known quantity and can improve + model error rates. + +commit de5713fa4b4a1b70d5fd4a697404361b52432dc7 +Author: Tom Veasey +Date: Tue Jan 14 17:35:00 2020 +0000 + + [ML] Disable invalid assertion (#50988) + + Backport #50986. + +commit ca9ca68cbebc506112a3c51dbc0acd3fa304625f +Author: Jason Tedor +Date: Tue Jan 14 12:12:20 2020 -0500 + + Allow installing multiple plugins as a transaction (#50924) + + This commit allows the plugin installer to install multiple plugins in a + single invocation. The installation will be treated as a transaction, so + that all of the plugins are install successfully, or none of the plugins + are installed. + +commit 16c07472e5d5039fd2f58e019df2dc714d534e56 +Author: Armin Braun +Date: Tue Jan 14 18:15:07 2020 +0100 + + Track Snapshot Version in RepositoryData (#50930) (#50989) + + * Track Snapshot Version in RepositoryData (#50930) + + Add tracking of snapshot versions to RepositoryData to make BwC logic more efficient. + Follow up to #50853 + +commit 7f309a18f114ec83f4478089c1519264162af88d +Author: David Kyle +Date: Tue Jan 14 17:14:39 2020 +0000 + + [7.x][ML] Explicitly require a OriginSettingClient in ML results iterators (#50981) + + In classes where the client is used directly rather than through a call to + executeAsyncWithOrigin explicitly require the client to be OriginSettingClient + rather than using the Client interface. + + Also remove calls to deprecated ClientHelper.clientWithOrigin() method. + +commit a5a8b60d7861333a2c8e23ec6978995d0195b964 +Author: Lisa Cawley +Date: Tue Jan 14 09:00:26 2020 -0800 + + [DOCS] Fix realm chains example (#50568) + +commit 6e7478b8461699093be6dd0162512ae5d28393f2 +Author: Tim Brooks +Date: Tue Jan 14 10:57:44 2020 -0600 + + Allow proxy mode server name to be configured (#50951) + + Currently, proxy mode allows a remote cluster connection to be setup by + expecting all open connections to be routed through an intermediate + proxy. The proxy must use some logic to ensure that the connections end + up on the correct remote cluster. One mechanism provided is that the + default distribution TLS implementations will forward the host component + of the configured address to the remote connection using the SNI + extension. This is limiting as it requires that the proxy be configured + in a way that always uses a valid hostname as the proxy address. + + Instead, this commit adds an additional setting to allow the server_name + to be configured independently. This allows the proxy address to be + specified as a IP literal, but the server_name specified as an arbitrary + string which still must be a valid hostname. It also decouples the + server_name from the requirement of being a DNS resolvable domain. + +commit 1fe2d76a91d9cb0a57f1767f719cab1d78337165 +Author: Armin Braun +Date: Tue Jan 14 17:46:47 2020 +0100 + + Fix S3 3rd Party Tests (#50983) + + Only load fixtures plugin in snapshot-tool tests if we're actually going to use a fixture + because otherwise configuration fails. + + Closes #50971 + +commit 6e8ea7aaa2fc3f35ffb6958e7113d304e23bc633 +Author: Armin Braun +Date: Tue Jan 14 17:13:21 2020 +0100 + + Work around JVM Bug in LongGCDisruptionTests (#50731) (#50974) + + There is a JVM bug causing `Thread#suspend` calls to randomly take + multiple seconds breaking these tests that call the method numerous times + in a loop. Increasing the timeout would will not work since we may call + `suspend` tens if not hundreds of times and even a small number of them + experiencing the blocking will lead to multiple minutes of waiting. + + This PR detects the specific issue by timing the `Thread#suspend` calls and + skips the remainder of the test if it timed out because of the JVM bug. + + Closes #50047 + +commit d8510be3d98b15f85056c84577122db549858100 +Author: Tim Brooks +Date: Tue Jan 14 09:53:13 2020 -0600 + + Revert "Send cluster name and discovery node in handshake (#48916)" (#50944) + + This reverts commit 0645ee88e2a3fff562a055fba2eaf928653c0db3. + +commit 4974f56b259c4787900d7dfcd7728a95dbba61f8 +Author: Alan Woodward +Date: Tue Jan 14 14:48:23 2020 +0000 + + Fix analysis BWC tests - warnings now emitted on index creation + +commit 1d8cb3c741650b54f9b7cacf514e0af5b25b768b +Author: Dimitris Athanasiou +Date: Tue Jan 14 16:46:09 2020 +0200 + + [7.x][ML] Add num_top_feature_importance_values param to regression and classi… (#50914) (#50976) + + Adds a new parameter to regression and classification that enables computation + of importance for the top most important features. The computation of the importance + is based on SHAP (SHapley Additive exPlanations) method. + + Backport of #50914 + +commit 0178c7c5d0baf636e778b82a8f186585a1226991 +Author: Hendrik Muhs +Date: Tue Jan 14 15:09:14 2020 +0100 + + [7.x][Transform] correctly retrieve checkpoints from remote indices (#50903) (#50969) + + uses remote client(s) to correctly retrieve index checkpoints from remote clusters + +commit f1c503176675a3b0a937c3595d2c4dd846adf054 +Author: Yannick Welsch +Date: Tue Jan 14 14:08:31 2020 +0100 + + Fix queuing in AsyncLucenePersistedState (#50958) + + The logic in AsyncLucenePersistedState was flawed, unexpectedly queuing up two update tasks + in parallel. + +commit 91d7b446a0c9e427b2f732cc724470ecd0eff7c6 +Author: Yannick Welsch +Date: Tue Jan 14 14:07:52 2020 +0100 + + Warn on slow metadata performance (#50956) + + Has the new cluster state storage layer emit warnings in case metadata performance is very + slow. + + Relates #48701 + +commit 8c16725a0d43d3ef154cd6fe4e85ee7751b8f0f5 +Author: Alan Woodward +Date: Tue Jan 14 13:12:25 2020 +0000 + + Check for deprecations when analyzers are built (#50908) + + Generally speaking, deprecated analysis components in elasticsearch will issue deprecation + warnings when they are first used. However, this means that no warnings are emitted when + indexes are created with deprecated components, and users have to actually index a document + to see warnings. This makes it much harder to see these warnings and act on them at + appropriate times. + + This is worse in the case where components throw exceptions on upgrade. In this case, users + will not be aware of a problem until a document is indexed, instead of at index creation time. + + This commit adds a new check that pushes an empty string through all user-defined analyzers + and normalizers when an IndexAnalyzers object is built for each index; deprecation warnings + and exceptions are now emitted when indexes are created or opened. + + Fixes #42349 + +commit 9c6ffdc2be0efb9a38d9f62a1b9371503ecd57f0 +Author: Przemysław Witek +Date: Tue Jan 14 14:43:39 2020 +0100 + + [7.x] Handle nested and aliased fields correctly when copying mapping. (#50918) (#50968) + +commit f028ab08d16ca0f6c5ce65b68157bb3794668033 +Author: James Rodewig +Date: Tue Jan 14 08:21:32 2020 -0500 + + [DOCS] Use `s` parameter in cat API overview example (#50616) + + Updates a snippet to use the `s` query string parameter rather than + piping the output to a separate `sort` command. + + This ensures the snippet is tested and available in clients other than + curl (Kibana console, etc.). + + Issue was originally raised by @hackaholic in #40926. + +commit 69a3626ee1d04570069a35cb1faf076a592600c9 +Author: David Kyle +Date: Thu Dec 19 10:52:54 2019 +0000 + + Mute SnapshotLifecycleRestIT testFullPolicySnapshot + + Relates to #50358 + +commit 277798606b19a36b26e88310e675f5fb117334d8 +Author: Florian Kelbert +Date: Tue Jan 14 11:28:19 2020 +0100 + + [DOCS] Correctly read total hits inside watcher config + + Relates to #50611 and #50612 + +commit 263083b882006f9e33f5fb639084c1aa1c2b00e2 +Author: Daniel Mitterdorfer +Date: Tue Jan 14 12:40:34 2020 +0100 + + Mute HttpCertificateCommandTests.testTextFileSubstitutions (#50965) (#50966) + + Relates #50964 + +commit 2bb7b53e419a1ebdb2f5bf529f48812aabc26260 +Author: Tim Vernum +Date: Tue Jan 14 21:24:21 2020 +1100 + + Add certutil http command (#50952) + + This adds a new "http" sub-command to the certutil CLI tool. + + The http command generates certificates/CSRs for use on the http + interface of an elasticsearch node/cluster. + It is designed to be a guided tool that provides explanations and + sugestions for each of the configuration options. The generated zip + file output includes extensive "readme" documentation and sample + configuration files for core Elastic products. + + Backport of: #49827 + +commit 22ba759e1f37ef2607674074973a554f23730b8f +Author: Yannick Welsch +Date: Tue Jan 14 09:35:43 2020 +0100 + + Move metadata storage to Lucene (#50928) + + * Move metadata storage to Lucene (#50907) + + Today we split the on-disk cluster metadata across many files: one file for the metadata of each + index, plus one file for the global metadata and another for the manifest. Most metadata updates + only touch a few of these files, but some must write them all. If a node holds a large number of + indices then it's possible its disks are not fast enough to process a complete metadata update before timing out. In severe cases affecting master-eligible nodes this can prevent an election + from succeeding. + + This commit uses Lucene as a metadata storage for the cluster state, and is a squashed version + of the following PRs that were targeting a feature branch: + + * Introduce Lucene-based metadata persistence (#48733) + + This commit introduces `LucenePersistedState` which master-eligible nodes + can use to persist the cluster metadata in a Lucene index rather than in + many separate files. + + Relates #48701 + + * Remove per-index metadata without assigned shards (#49234) + + Today on master-eligible nodes we maintain per-index metadata files for every + index. However, we also keep this metadata in the `LucenePersistedState`, and + only use the per-index metadata files for importing dangling indices. However + there is no point in importing a dangling index without any shard data, so we + do not need to maintain these extra files any more. + + This commit removes per-index metadata files from nodes which do not hold any + shards of those indices. + + Relates #48701 + + * Use Lucene exclusively for metadata storage (#50144) + + This moves metadata persistence to Lucene for all node types. It also reenables BWC and adds + an interoperability layer for upgrades from prior versions. + + This commit disables a number of tests related to dangling indices and command-line tools. + Those will be addressed in follow-ups. + + Relates #48701 + + * Add command-line tool support for Lucene-based metadata storage (#50179) + + Adds command-line tool support (unsafe-bootstrap, detach-cluster, repurpose, & shard + commands) for the Lucene-based metadata storage. + + Relates #48701 + + * Use single directory for metadata (#50639) + + Earlier PRs for #48701 introduced a separate directory for the cluster state. This is not needed + though, and introduces an additional unnecessary cognitive burden to the users. + + Co-Authored-By: David Turner + + * Add async dangling indices support (#50642) + + Adds support for writing out dangling indices in an asynchronous way. Also provides an option to + avoid writing out dangling indices at all. + + Relates #48701 + + * Fold node metadata into new node storage (#50741) + + Moves node metadata to uses the new storage mechanism (see #48701) as the authoritative source. + + * Write CS asynchronously on data-only nodes (#50782) + + Writes cluster states out asynchronously on data-only nodes. The main reason for writing out + the cluster state at all is so that the data-only nodes can snap into a cluster, that they can do a + bit of bootstrap validation and so that the shard recovery tools work. + Cluster states that are written asynchronously have their voting configuration adapted to a non + existing configuration so that these nodes cannot mistakenly become master even if their node + role is changed back and forth. + + Relates #48701 + + * Remove persistent cluster settings tool (#50694) + + Adds the elasticsearch-node remove-settings tool to remove persistent settings from the on + disk cluster state in case where it contains incompatible settings that prevent the cluster from + forming. + + Relates #48701 + + * Make cluster state writer resilient to disk issues (#50805) + + Adds handling to make the cluster state writer resilient to disk issues. Relates to #48701 + + * Omit writing global metadata if no change (#50901) + + Uses the same optimization for the new cluster state storage layer as the old one, writing global + metadata only when changed. Avoids writing out the global metadata if none of the persistent + fields changed. Speeds up server:integTest by ~10%. + + Relates #48701 + + * DanglingIndicesIT should ensure node removed first (#50896) + + These tests occasionally failed because the deletion was submitted before the + restarting node was removed from the cluster, causing the deletion not to be + fully acked. This commit fixes this by checking the restarting node has been + removed from the cluster. + + Co-authored-by: David Turner + + * fix tests + + Co-authored-by: David Turner + +commit b02b073a57a82186d11e67d57734d7319403fcfb +Author: Tim Vernum +Date: Tue Jan 14 18:04:02 2020 +1100 + + Increase Size and lower TTL on DLS BitSet Cache (#50953) + + The Document Level Security BitSet Cache (see #43669) had a default + configuration of "small size, long lifetime". However, this is not + a very useful default as the cache is most valuable for BitSets that + take a long time to construct, which is (generally speaking) the same + ones that operate over a large number of documents and contain many + bytes. + + This commit changes the cache to be "large size, short lifetime" so + that it can hold bitsets representing billions of documents, but + releases memory quickly. + + The new defaults are 10% of heap, and 2 hours. + + This also adds some logging when a single BitSet exceeds the size of + the cache and when the cache is full. + + Backport of: #50535 + +commit 33c29fb5a3c91a3110e82cd1ca4fa37846ace624 +Author: Tim Vernum +Date: Tue Jan 14 15:26:41 2020 +1100 + + Support Client and RoleMapping in custom Realms (#50950) + + Previously custom realms were limited in what services and components + they had easy access to. It was possible to work around this because a + security extension is packaged within a Plugin, so there were ways to + store this components in static/SetOnce variables and access them from + the realm, but those techniques were fragile, undocumented and + difficult to discover. + + This change includes key services as an argument to most of the methods + on SecurityExtension so that custom realm / role provider authors can + have easy access to them. + + Backport of: #50534 + +commit 50cb7703156477f56265fbb4a005f0eab5737902 +Author: Tim Brooks +Date: Mon Jan 13 21:46:23 2020 -0600 + + Use default profile for remote connections (#50947) + + Currently, the connection manager is configured with a default profile + for both the sniff and proxy connection stratgies. This profile + correctly reflects the expected number of connection (6 for sniff, 18 + for proxy). This commit removes the proxy strategy usages of the per + connection attempt profile configuration. + + Additionally, it refactors other unnecessary code around the connection + manager. The connection manager now can always be built inside the + remote connection. + +commit 90ba77951a2cbeb30979490cf3aa7cb42296526b +Author: Tim Vernum +Date: Tue Jan 14 13:19:05 2020 +1100 + + Fix memory leak in DLS bitset cache (#50946) + + The Document Level Security BitSet cache stores a secondary "lookup + map" so that it can determine which cache entries to invalidate when + a Lucene index is closed (merged, etc). + + There was a memory leak because this secondary map was not cleared + when entries were naturally evicted from the cache (due to size/ttl + limits). + + This has been solved by adding a cache removal listener and processing + those removal events asyncronously. + + Backport of: #50635 + +commit 27c2eb744ec6f5ed91deb6437c393c0fbb02e977 +Author: Tim Brooks +Date: Wed Jan 8 14:33:11 2020 -0600 + + Fix open/close race in ConnectionManagerTests (#50621) + + Currently we reuse the same test connection for all connection attempts + in the testConcurrentConnectsAndDisconnects test. This means that if the + connection fails due to a pre-existing connection, the connection will + be closed impacting the state of all connection attempts. This commit + fixes the test, by returning a unique connection for each attempt. + + Fixes #49903. + +commit 1577a0e617cff31d2e3e739aaf1b6b57c8590366 +Author: Tim Vernum +Date: Tue Jan 14 12:37:45 2020 +1100 + + Validate field permissions when creating a role (#50917) + + When creating a role, we do not check if the exceptions for + the field permissions are a subset of granted fields. If such + a role is assigned to a user then that user's authentication fails + for this reason. + + We added a check to validate role query in #46275 and on the same lines, + this commit adds check if the exceptions for the field + permissions is a subset of granted fields when parsing the + index privileges from the role descriptor. + + Backport of: #50212 + + Co-authored-by: Yogesh Gaikwad + +commit c2acb8830a48e767b630e365e808a1d030357045 +Author: Tim Vernum +Date: Tue Jan 14 12:37:05 2020 +1100 + + Add max_resource_units to enterprise license (#50910) + + The enterprise license type must have "max_resource_units" and may not + have "max_nodes". + + This change adds support for this new field, validation that the field + is present if-and-only-if the license is enterprise and bumps the + license version number to reflect the new field. + + Includes a BWC layer to return "max_nodes: ${max_resource_units}" in + the GET license API. + + Backport of: #50735 + +commit f0924e6d5bbd88d00569c320195d5b31977c896b +Author: Nhat Nguyen +Date: Mon Jan 13 19:59:50 2020 -0500 + + Remove outdated requirement of CCR (#50859) + + With retention leases, users do not need to set + index.soft_deletes.retention.operations. This change removes it from the + requirements of CCR + +commit fb32a55dd5fc108932855724ec77ecb5bf1d7a65 +Author: Nhat Nguyen +Date: Mon Jan 13 10:58:29 2020 -0500 + + Deprecate synced flush (#50835) + + A normal flush has the same effect as a synced flush on Elasticsearch + 7.6 or later. It's deprecated in 7.6 and will be removed in 8.0. + + Relates #50776 + +commit a18736b46d226bb3cb16efaf63b5c75cc52230d6 +Author: Przemko Robakowski +Date: Tue Jan 14 01:34:33 2020 +0100 + + [7.x] ILM action to wait for SLM policy execution (#50454) (#50943) + + * ILM action to wait for SLM policy execution (#50454) + + This change add new ILM action to wait for SLM policy execution to ensure that index has snapshot before deletion. + + Closes #45067 + + * Fix flaky TimeSeriesLifecycleActionsIT#testWaitForSnapshot test + + This change adds some randomness and cleanup step to TimeSeriesLifecycleActionsIT#testWaitForSnapshot and testWaitForSnapshotSlmExecutedBefore tests in attempt to make them stable. + + Reletes to #50781 + + * Formatting changes + + * Longer timeout + + * Fix Map.of in Java8 + + * Unused import removed + +commit 4cb525d8d3876c94b0d3f161de1c7bdef412210e +Author: Peter Dyson +Date: Tue Jan 14 09:44:35 2020 +1000 + + [DOCS] Array of index patterns is also valid source indices with transform (#50777) + +commit 86fb06a1084254e8b37e39f07c6168cd807253db +Author: Ryan Ernst +Date: Mon Jan 13 13:56:30 2020 -0800 + + Migrate certgen packaging test from bats (#50880) + + This commit moves the packaging tests for elasticsearch-certgen + to java from bats. Although certgen is deprecated, the tests are + moved rather than just deleted, and the tests themselves should be + easily adaptable to certutil. One note is that the test is simplified to + use a single node, rather than the two node test from bats, which was + problematic given how the newer distro tests only operate with a single + distribution. + + relates #46005 + +commit 91689e793dabd5a8e9013bb9d68ca9623f3a4eea +Author: Lee Hinman +Date: Mon Jan 13 14:31:41 2020 -0700 + + [7.x] Refresh cached phase policy definition if possible on ne… (#50941) + + * Refresh cached phase policy definition if possible on new policy + + There are some cases when updating a policy does not change the + structure in a significant way. In these cases, we can reread the + policy definition for any indices using the updated policy. + + This commit adds this refreshing to the `TransportPutLifecycleAction` + to allow this. It allows us to do things like change the configuration + values for a particular step, even when on that step (for example, + changing the rollover criteria while on the `check-rollover-ready` step). + + There are more cases where the phase definition can be reread that just + the ones checked here (for example, removing an action that has already + been passed), and those will be added in subsequent work. + + Relates to #48431 + +commit a82ddfb182f6de033aac5609e945308599912056 +Author: Lisa Cawley +Date: Mon Jan 13 13:02:06 2020 -0800 + + [DOCS] Adds elasticsearch-keystore command reference (#50872) + +commit f04b4cbee8e8be2f53afe17d8136b62688f78125 +Author: Bogdan Pintea +Date: Mon Jan 13 21:51:29 2020 +0100 + + SQL: Optimisation fixes for conjunction merges (#50703) (#50933) + + * SQL: Optimisation fixes for conjunction merges + + This commit fixes the following issues around the way comparisions are + merged with ranges in conjunctions: + * the decision to include the equality of the lower limit is corrected; + * the selection of the upper limit is corrected to use the upper bound + of the range; + * the list of terms in the conjunction is sorted to have the ranges at + the bottom; this allows subsequent binary comarisions to find compatible + ranges and potentially be merged away. The end guarantee being that the + optimisation takes place irrespective of the order of the conjunction + terms in the statement. + + Some comments are also corrected. + + * adress review observation on anon. comparator + + Replace anonymous comparator of split AND Expressions with a lambda. + + (cherry picked from commit 9828cb143a41f1bda1219541f3a8fdc03bf6dd14) + +commit 2dc23bd9686bc59f2098e2195db0d105db414633 +Author: Ryan Ernst +Date: Mon Jan 13 10:38:25 2020 -0800 + + Add protection in windows for slow file lock releasing (#50884) + + This commit adds retries for windows cleanup after tests, which may fail + due to file locks not being immediately released after a windows process + exits. + + closes #50825 + +commit 56bbed3593fc4b159798ba640b1e6cf96285473b +Author: Mark Vieira +Date: Mon Jan 13 09:31:39 2020 -0800 + + Fix build scan logic to support folder nested Jenkins job names + + Signed-off-by: Mark Vieira + +commit 05f97d5e1bdc8ab66077a4dc0496071f391c8bc6 +Author: Nhat Nguyen +Date: Mon Jan 13 11:40:47 2020 -0500 + + Revert "Deprecate synced flush (#50835)" + + This reverts commit 1a32d7142aa374b38707266588594f254af31014. + +commit d1bde0a7189ec1f12496ad334907c8c16304538c +Author: Darren Foong +Date: Mon Jan 13 16:11:52 2020 +0000 + + Improve warning value extraction performance in Response (#50208) + + This commit improves the performance of warning value extraction in the + low-level REST client, and is similar to the approach taken in + #24114. There are some differences since the low-level REST client might + be connected to Elasticsearch through a proxy that injects its own + warnings. + +commit 1a32d7142aa374b38707266588594f254af31014 +Author: Nhat Nguyen +Date: Mon Jan 13 10:58:29 2020 -0500 + + Deprecate synced flush (#50835) + + A normal flush has the same effect as a synced flush on Elasticsearch + 7.6 or later. It's deprecated in 7.6 and will be removed in 8.0. + + Relates #50776 + +commit ba37e3c4a092141ee869bf05cdcc44709bd433e0 +Author: Ioannis Kakavas +Date: Mon Jan 13 17:04:23 2020 +0200 + + Disable DiagnosticTrustManager in FIPS 140 (#49888) + + This commit changes the default behavior for + xpack.security.ssl.diagnose.trust when running in a FIPS 140 JVM. + + More specifically, when xpack.security.fips_mode.enabled is true: + + - If xpack.security.ssl.diagnose.trust is not explicitly set, the + default value of it becomes false and a log message is printed + on info level, notifying of the fact that the TLS/SSL diagnostic + messages are not enabled when in a FIPS 140 JVM. + - If xpack.security.ssl.diagnose.trust is explicitly set, the value of + it is honored, even in FIPS mode. + + This is relevant only for 7.x where we support Java 8 in which + SunJSSE can still be used as a FIPS 140 provider for TLS. SunJSSE + in FIPS mode, disallows the use of other TrustManager implementations + than the one shipped with SunJSSE. + +commit 6718ce0f627a22f82b9e613bdbbdf529366517df +Author: junmuz +Date: Mon Jan 13 19:47:36 2020 +0500 + + [DOCS] Correct typo in `ignore_malformed` mapping parm docs (#50780) + +commit 609b015e3c1c18ef8e31bd08d804d79ee28b8c00 +Author: Armin Braun +Date: Mon Jan 13 15:02:53 2020 +0100 + + Prevent Old Version Clusters From Corrupting Snapshot Repositories (#50853) (#50913) + + Follow up to #50692 that starts writing a `min_version` field to + the `RepositoryData` so that pre-7.6 ES versions can not read it + (and potentially corrupt it if they attempt to modify the repo contents) + after the repository moved to the new metadata format. + +commit cc8aafcfc2804e2783a7b832027fce04a8058281 +Author: Larry Gregory +Date: Mon Jan 13 08:36:48 2020 -0500 + + [7.x] - Adding GET/PUT ILM cluster privileges to `kibana_syste… (#50878) + + Co-authored-by: Elastic Machine + + Co-authored-by: Elastic Machine + +commit eb8fd448368cc1154e7ab8d5416dbbe3c4e40a2d +Author: Benjamin Trent +Date: Mon Jan 13 07:59:31 2020 -0500 + + [ML][Inference] minor fixes for created_by, and action permission (#50890) (#50911) + + The system created and models we provide now use the `_xpack` user for uniformity with our other features + + The `PUT` action is now an admin cluster action + + And XPackClient class now references the action instance. + +commit 4e837599b3ee80ec205985dc6914e8bccae2d05e +Author: Albert Zaharovits +Date: Mon Jan 13 13:25:05 2020 +0200 + + Nit fix test randomInt bound + + Relates 2b789fa3e6f743632066 + +commit 2b789fa3e6f743632066b26efd5235d3acdd2264 +Author: Albert Zaharovits +Date: Mon Jan 13 12:20:54 2020 +0200 + + Make .async-search-* a restricted namespace (#50294) + + Hide the `.async-search-*` in Security by making it a restricted index namespace. + The namespace is hard-coded. + To grant privileges on restricted indices, one must explicitly toggle the + `allow_restricted_indices` flag in the indices permission in the role definition. + As is the case with any other index, if a certain user lacks all permissions for an + index, that index is effectively nonexistent for that user. + +commit c31a21c3d892fa3aa4c7c6e78a7d70a8c083df4b +Author: Christoph Büscher +Date: Mon Jan 13 10:10:15 2020 +0100 + + Fix time zone issue in Rounding serialization (#50845) + + When deserializing time zones in the Rounding classes we used to include a tiny + normalization step via `DateUtils.of(in.readString())` that was lost in #50609. + Its at least necessary for some tests, e.g. the cause of #50827 is that when + sending the default time zone ZoneOffset.UTC on a stream pre 7.0 we convert it + to a "UTC" string id via `DateUtils.zoneIdToDateTimeZone`. This gets then read + back as a UTC ZoneRegion, which should behave the same but fails the equality + tests in our serialization tests. Reverting to the previous behaviour with an + additional normalization step on 7.x. + + Co-authored-by: Nik Everett + + Closes #50827 + +commit 985c95dccace0e6a31acc60d201bfddc0ad5c068 +Author: Tim Vernum +Date: Mon Jan 13 18:02:22 2020 +1100 + + Populate OpenIDConnect metadata collections (#50893) + + The OpenIdConnectRealm had a bug which would cause it not to populate + User metadata for collections contained in the user JWT claims. + + This commit fixes that bug. + + Backport of: #50521 + +commit fa116a6d266b94eb5ceeaa8c6b2e0e5dd4074e17 +Author: Benjamin Trent +Date: Sun Jan 12 10:59:11 2020 -0500 + + [7.x] [ML][Inference] PUT API (#50852) (#50887) + + * [ML][Inference] PUT API (#50852) + + This adds the `PUT` API for creating trained models that support our format. + + This includes + + * HLRC change for the API + * API creation + * Validations of model format and call + + * fixing backport + +commit 456de59698f9487d374e1ae8116a23c0cd91a736 +Author: David Turner +Date: Sun Jan 12 12:37:52 2020 +0000 + + Fix non-corruption in testCurrentHeaderVersion (#50883) + + Today we make multiple attempts to corrupt the translog header in + `TranslogHeaderTests#testCurrentHeaderVersion`, but if we are extraordinarily + unlucky then this sequence of corruptions may restore the file to its original + state. This change adjusts the test to only corrupt the file once, which is + certain not to leave the file in its original state. + +commit 2e5e5fd483a0ccb081c81cd44615b5a833bfd763 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jan 10 17:29:00 2020 +0100 + + Fix testSkipRefreshIfShardIsRefreshingAlready (#50856) + + The test checked queue size and active count, however, + ThreadPoolExecutor pulls out the request from the queue before marking + the worker active, risking that we think all tasks are done when they + are not. Now check on completed-tasks metric instead, which is + guaranteed to be monotonic. + + Relates #50769 + +commit f4aabdcd89b701a38bc5c79cb1dd38797c0a77d1 +Author: Nhat Nguyen +Date: Thu Jan 9 18:18:23 2020 -0500 + + Do not force refresh when write indexing buffer (#50769) + + Today we periodically check the indexing buffer memory every 5 seconds + or after we have used 1/30 of the configured memory. If the total used + memory is over the threshold, then we refresh the "largest" shards. If + refreshing takes longer these intervals (i.e., 5s or 1/30 buffer), then + we continue to enqueue refreshes to these shards. This leads to two + issues: + + - The refresh thread pool can be exhausted and other shards can't refresh + - Execute too many refreshes for the "largest" shards + + With this change, we only refresh the largest shards if they are not refreshing. + Here we rely on the periodic check to trigger another refresh if needed. We can + harden this by making the ongoing refresh triggers the memory check when + it's completed. I opted out this option in this PR for simplicity. + + See: https://discuss.elastic.co/t/write-queue-continue-to-rise/213652/ + +commit 981e4b8d17d03dec83f8c278f6d93688ac9787b3 +Author: Mark Vieira +Date: Fri Jan 10 14:51:25 2020 -0800 + + Improve build scan tags and values for Jenkins matrix jobs (#50881) + +commit 63472d30c756c9c38d641e29017be865bdd1edfa +Author: Lee Hinman +Date: Fri Jan 10 14:27:55 2020 -0700 + + [7.x] Fix SLM check for restore in progress (#50868) (#50876) + + * Fix SLM check for restore in progress (#50868) + + * Fix SLM check for restore in progress + + This commit fixes the check in SLM where the `RestoreInProgress` + metadata was checked for existence. Rather than check existence we + should instead check the `isEmpty` method. Prior to this, a successful + restore for a repository that used SLM retention would prevent SLM + retention from running in subsequent invocations, due to SLM thinking + that a restore was still running. + + * Fix 7.x-isms + +commit e6d0f7df01c95eb3dfdadc0a797764ba0c90052b +Author: Nik Everett +Date: Fri Jan 10 16:18:11 2020 -0500 + + Fix format problem in composite of unmapped (#50869) (#50875) + + When a composite aggregation is reduced using the results from an + index that has one of the fields unmapped we were throwing away the + formatter. This is mildly annoying, except in the case of IP addresses + which were coming out as non-utf-8-characters. And tripping assertions. + + This carefully preserves the formatter from the working bucket. + + Closes #50600 + +commit 3bac1dc414ab1437fbaa7b50e64d1987f754411f +Author: Julie Tibshirani +Date: Fri Jan 10 09:36:02 2020 -0800 + + Adjust the skip version in flattened field telemetry tests. + + We forgot to adjust the version when backporting the commit to 7.x. + +commit 60308cf0b35c2ed0f10aefadc91f418922b1f9af +Author: Jim Ferenczi +Date: Fri Jan 10 17:51:56 2020 +0100 + + Fix upgrade of custom similarity (#50851) + + This change fixes the upgrade of index metadata that contain + a custom similarity with options that are not compatible with BM25. + The upgrade doesn't need a real similarity service so we fake one that + resolves all custom similarity to BM25 but this logic fails because the + BM25 provider checks that all options are compatible. This commit removes + the verification step as it is not needed during the upgrade (the verification + is done when the index is restored/opened). + + Closes #50763 + +commit ae8308252b69e116f47ab82365f5e1c47c2b8bed +Author: Julie Tibshirani +Date: Fri Jan 10 09:26:53 2020 -0800 + + Ensure we emit a warning when using the deprecated 'template' field. (#50831) + + Since 6.0, the 'template' field has been deprecated in put template requests in + favour of index_patterns. Previously, the PutIndexTemplateRequest would accept + the 'template' field in its 'source' methods and silently convert it to + 'index_patterns'. This meant that users specifying 'template' in the source + would not receive a deprecation warning from the server. + + This PR makes a small change to no longer silently convert 'template' to + 'index_patterns', which ensures that users receive a deprecation warning. + + Follow-up to #49460. + +commit 5afa0b71e914551b8447689b711186571a4b5f94 +Author: Benjamin Trent +Date: Fri Jan 10 12:00:37 2020 -0500 + + [ML][Inference] Unify top_classes object field names with analytics (#50858) (#50861) + +commit 4629a9714c79dc467a1360ebf79e06af8fd9c799 +Author: James Rodewig +Date: Fri Jan 10 07:23:25 2020 -0600 + + [DOCS] Fix time_zone example in range query docs (#50830) + + One of the example snippets in the range query docs was missing a + required 'T' in the `date` format. This adds the required 'T'. + +commit 48e5eece1e7343efe3073aafe234b408262ea759 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jan 10 13:54:32 2020 +0100 + + GlobalBuildInfo support packed-refs with work-tree (#50791) + + The packed-refs support was using the original .git path, changed to use + the real .git directory after reference from worktree has been followed. + + Relates #47464 + +commit 422422a2bc6b15ca5f2f118eb3b01dab915d1750 +Author: Dimitris Athanasiou +Date: Fri Jan 10 14:24:13 2020 +0200 + + [7.x][ML] Reuse SourceDestValidator for data frame analytics (#50841) (#50850) + + This commit removes validation logic of source and dest indices + for data frame analytics and replaces it with using the common + `SourceDestValidator` class which is already used by transforms. + This way the validations and their messages become consistent + while we reduce code. + + This means that where these validations fail the error messages + will be slightly different for data frame analytics. + + Backport of #50841 + +commit 7e68989dae13a815f0fba2b8b6deecaa72c4b47d +Author: Armin Braun +Date: Fri Jan 10 11:49:52 2020 +0100 + + Fix Snapshot Shard Status Request Deduplication (#50788) (#50840) + + * Fix Snapshot Shard Status Request Deduplication + + The request deduplication didn't actually work for these requests + since they had no `equals` and `hashCode` so the deduplicator wouldn't + actually recognize equal requests. + +commit 75cb4e0b69a1446b2fd8b3b45a2bc58ca622c056 +Author: Christoph Büscher +Date: Fri Jan 10 09:24:09 2020 +0100 + + Muting InternalAggregationsTests.testSerialization + +commit 83d961391bc51c3df02d076bd6e0a82ada091ffb +Author: debadair +Date: Thu Jan 9 16:55:46 2020 -0800 + + [DOCS] Move snapshot-restore out of modules. (#49618) (#50829) + + * [DOCS] Move snapshot-restore docs out of modules. + + * [DOCS] Incorporates comments from @jrodewig. + + * [DOCS] Fix snippet tests + +commit 2ab3e49dec6fd91dbc5452f650bf5e5498f58af0 +Author: Mark Vieira +Date: Thu Jan 9 15:43:44 2020 -0800 + + Skip test suite entirely for non-applicable distribution types (#50824) + +commit a9c4db4134bd564523415a5f3a264860ac031fdf +Author: Julie Tibshirani +Date: Thu Jan 9 15:25:46 2020 -0800 + + Correct the skip version in tests for field collapsing on an alias. + +commit d021071ab935cf4f09f39c530e5e368e152c2eed +Author: Nik Everett +Date: Thu Jan 9 16:09:21 2020 -0500 + + Move scripted metric to ObjectParser (#50708) (#50811) + + This replaces the hand rolled parsing code for scripted metric with + `ObjectParser` which is simpler to work with because it is declarative. + +commit ae40e2245212220e2706c05d8eeda6db93e67908 +Author: Nik Everett +Date: Thu Jan 9 15:53:03 2020 -0500 + + Drop "funny" functions building parsers (#50715) (#50814) + + Replaces the "funny" + `Function>` with a much + simpler `ConstructingObjectParser`. This makes pretty much + all of our object parsers static. + +commit f70e8f6ab559a53081778f26e60658220624cd5f +Author: Armin Braun +Date: Thu Jan 9 21:21:13 2020 +0100 + + Fix Snapshot Repository Corruption in Downgrade Scenarios (#50692) (#50797) + + * Fix Snapshot Repository Corruption in Downgrade Scenarios (#50692) + + This PR introduces test infrastructure for downgrading a cluster while interacting with a given repository. + It fixes the fact that repository metadata in the new format could be written while there's still older snapshots in the repository that require the old-format metadata to be restorable. + +commit 344c21813be368dab00993373be210d6ac4621b5 +Author: Matt Braymer-Hayes +Date: Thu Jan 9 14:35:05 2020 -0500 + + [DOCS] Fix typo in refresh API docs (#50759) + +commit ef1c14ad01edb818d2bf35075914a6bb4bee772d +Author: Lisa Cawley +Date: Thu Jan 9 11:26:34 2020 -0800 + + [DOCS] Update license expiry links (#50812) + +commit de6f132887c3fce21482002f432f9802feed3db8 +Author: Jake Landis +Date: Thu Jan 9 13:21:18 2020 -0600 + + [7.x] Foreach processor - fork recursive call (#50514) (#50773) + + A very large number of recursive calls can cause a stack overflow + exception. This commit forks the recursive calls for non-async + processors. Once forked, each thread will handle at most 10 + recursive calls to help keep the stack size and thread count + down to a reasonable size. + +commit 1d8e51f89d67f1df5df15743e1363eacef083f81 +Author: Nik Everett +Date: Thu Jan 9 14:11:24 2020 -0500 + + Support offset in composite aggs (#50609) (#50808) + + Adds support for the `offset` parameter to the `date_histogram` source + of composite aggs. The `offset` parameter is supported by the normal + `date_histogram` aggregation and is useful for folks that need to + measure things from, say, 6am one day to 6am the next day. + + This is implemented by creating a new `Rounding` that knows how to + handle offsets and delegates to other rounding implementations. That + implementation doesn't fully implement the `Rounding` contract, namely + `nextRoundingValue`. That method isn't used by composite aggs so I can't + be sure that any implementation that I add will be correct. I propose to + leave it throwing `UnsupportedOperationException` until I need it. + + Closes #48757 + +commit c51303d051b38553b359d7376d37159c7dbec9c8 +Author: Sean Story +Date: Thu Jan 9 09:48:20 2020 -0600 + + Typo of ' instead of ` (#50767) + +commit 8a5de4f56fd4b8ed1f4d17d60c5cc9b25b5e234d +Author: lcawl +Date: Thu Jan 9 08:34:34 2020 -0800 + + [DOCS] Clarify detector_index property in ML APIs (#50723) + +commit cc0e64572a4892e93139ec0a8904d0c470defc07 +Author: Benjamin Trent +Date: Thu Jan 9 10:33:38 2020 -0500 + + [ML][Inference][HLRC] Add necessary lang ident classes (#50705) (#50794) + + This adds the necessary named XContent classes to the HLRC for the lang ident model. This is so the HLRC can call `GET _ml/inference/lang_ident_model_1?include_definition=true` without XContent parsing errors. + + The constructors are package private as since this classes are used exclusively within the pre-packaged model (and require the specific weights, etc. to be of any use). + +commit 3e014d39c2d863149c0754da105f8b1c1cbfa5ca +Author: Benjamin Trent +Date: Thu Jan 9 10:33:22 2020 -0500 + + [Transform] fail to start/put on missing pipeline (#50701) (#50795) + + If a pipeline referenced by a transform does not exist, we should not allow the transform to be created. + + We do allow the pipeline existence check to be skipped with defer_validations, but if the pipeline still does not exist on `_start`, the pipeline will fail to start. + + relates: #50135 + +commit 4f150e4961f9cffe5b47e9e31299b910c8ff028f +Author: István Zoltán Szabó +Date: Thu Jan 9 16:21:35 2020 +0100 + + [7.x][DOCS] Moves analysis resources to PUT DFA API docs (#50793) + +commit 71afeec7d01973db8a3fc6e9f17e8034e908e395 +Author: István Zoltán Szabó +Date: Thu Jan 9 14:31:35 2020 +0100 + + Revert "[DOCS] Moves analysis resources to PUT DFA API docs (#50704)" + + This reverts commit 4e1107d5d717599ddf1632c3253de6f1df4a51af. + +commit 4e1107d5d717599ddf1632c3253de6f1df4a51af +Author: István Zoltán Szabó +Date: Thu Jan 9 13:57:11 2020 +0100 + + [DOCS] Moves analysis resources to PUT DFA API docs (#50704) + + Co-authored-by: Lisa Cawley + +commit acd73dda1c5c6462eb6ffb56c3cc44b2a1d72e17 +Author: István Zoltán Szabó +Date: Thu Jan 9 11:19:19 2020 +0100 + + [DOCS] Improves find_file_structure documentation (#50743) + + Co-authored-by: Lisa Cawley + +commit 4a7e09f62446e4622e19905fe497677eaee5e5d3 +Author: Armin Braun +Date: Thu Jan 9 10:46:07 2020 +0100 + + Enforce Logging of Errors in GCS Rest RetriesTests (#50761) (#50783) + + It's impossible to tell why #50754 fails without this change. + We're failing to close the `exchange` somewhere and there is no + write timeout in the GCS SDK (something to look into separately) + only a read timeout on the socket so if we're failing on an assertion without + reading the full request body (at least into the read-buffer) we're locking up + waiting forever on `write0`. + + This change ensure the `exchange` is closed in the tests where we could lock up + on a write and logs the failure so we can find out what broke #50754. + +commit 0ac6786f415e742d8202c97c1cae3bcefe67d499 +Author: István Zoltán Szabó +Date: Thu Jan 9 10:44:07 2020 +0100 + + [DOCS] Forms role and privilege requirements as bulleted lists in DFA API docs (#50732) + + Co-Authored-By: Lisa Cawley + +commit 8a1bb440e2d15e85a6b2f9f6949585bbd17e3c8f +Author: István Zoltán Szabó +Date: Thu Jan 9 09:43:55 2020 +0100 + + [DOCS] Clarifies model_size_stats.total_xxx_field_count objects and removes notes in GET job stats API docs. (#50728) + +commit d7bb5d7531a8a865d43ba17bcf6718512268800e +Author: István Zoltán Szabó +Date: Thu Jan 9 09:31:30 2020 +0100 + + [DOCS] Improves description for forecast_stats (#50729) + + Co-Authored-By: Lisa Cawley + +commit f75d99149b553e0e561498ff45ccfbd8853fc1fd +Author: Martijn van Groningen +Date: Thu Jan 9 09:01:55 2020 +0100 + + Wrap triggering of a watch inside an assertBusy(...) invocation + + This test replaces the watch index after watcher got started. + This triggers watches being reloaded and while this happens the + trigger engine is paused, which disallows watches from being + triggered. At this time there are no watches in the .watches + index and I think this is just unlucky timing. + + Reloading of watches happens in the background and + the watch state can be started when that happens. + For normal schedule trigger engines this is not an issue, + because watches that are meant to be triggered are triggered + when the engine triggers the next time. However for the + mock scheduled trigger engine this is different, + because watches are triggered programatically and + there is no retry in this test. + + I think just adding `timeWarp().trigger("mywatch");` inside + a `assertBusy(...)`` is the right fix here. If it fails + because the mock schedule trigger engine is paused then + the test will try again. In the mean time the the watches + can be reloaded, which then resumes the mock scheduled trigger engine. + + Closes #50658 + +commit d2189b9d8067a597548b7e364effed368eb9657d +Author: Ioannis Kakavas +Date: Thu Jan 9 09:41:04 2020 +0200 + + Mute SamlAuthenticatorTests in Azulu Zulu (#50779) + + See #49742 + +commit 8d9ec1c66dddb391ff75e91fc2995098d0acb2d7 +Author: Jason Tedor +Date: Wed Jan 8 16:51:41 2020 -0500 + + Remove UBI-based Docker images (#50747) + + This commit removes the UBI-based Docker images as we are not using this + work for now. + +commit 3273c366b51d244b9ae5641f811167fd64389314 +Author: Julie Tibshirani +Date: Wed Jan 8 15:57:52 2020 -0800 + + Adjust the skip version in tests for field collapsing with aliases. + + Follow-up to ##50766, which was also backported to 7.5. + +commit a299aba2f8445136087eaaffd07f5a69f0e9299e +Author: Julie Tibshirani +Date: Wed Jan 8 14:51:15 2020 -0800 + + Ensure that field collapsing works with field aliases. (#50766) + + Previously, the following situation would throw an error: + * A search contains a `collapse` on a particular field. + * The search spans multiple indices, and in one index the field is mapped as a + concrete field, but in another it is a field alias. + + The error occurs when we attempt to merge `CollapseTopFieldDocs` across shards. + When merging, we validate that the name of the collapse field is the same across + shards. But the name has already been resolved to the concrete field name, so it + will be different on shards where the field was mapped as an alias vs. shards + where it was a concrete field. + + This PR updates the collapse field name in `CollapseTopFieldDocs` to the + original requested field, so that it will always be consistent across shards. + + Note that in #32648, we already made a fix around collapsing on field aliases. + However, we didn't test this specific scenario where the field was mapped as an + alias in only one of the indices being searched. + +commit b1b42822730c97953e54666a4a45e8da64c95090 +Author: Christoph Büscher +Date: Wed Jan 8 18:25:00 2020 +0100 + + Make Multiplexer inherit filter chains analysis mode (#50662) + + Currently, if an updateable synonym filter is included in a multiplexer filter, + it is not reloaded via the _reload_search_analyzers because the multiplexer + itself doesn't pass on the analysis mode of the filters it contains, so its not + recognized as "updateable" in itself. Instead we can check and merge the + AnalysisMode settings of all filters in the multiplexer and use the resulting + mode (e.g. search-time only) for the multiplexer itself, thus making any synonym + filters contained in it reloadable. This, of course, will also make the + analyzers using the multiplexer be usable at search-time only. + + Closes #50554 + +commit 78c9eee5ea5f1db9b99bf6552e90ab24b342c014 +Author: James Rodewig +Date: Wed Jan 8 14:39:28 2020 -0600 + + [DOCS] Add section ID to analysis overview page + +commit 8dc6e988193d54814e271256d5cdf720a62a2549 +Author: Lee Hinman +Date: Wed Jan 8 13:13:57 2020 -0700 + + [7.x] Make InitializePolicyContextStep retryable (#50685) (#50760) + + This commits makes the "init" ILM step retryable. It also adds a test + where an index is created with a non-parsable index name and then fails. + + Related to #48183 + +commit e95b0c447f429a0799bfd0bdaad38bfe30d43a59 +Author: Przemyslaw Gomulka +Date: Wed Jan 8 20:04:57 2020 +0100 + + Allow parsing timezone without fully provided time backport(#50178) (#50740) + + strict_date_optional_time changes to have optional minute part. + It already allowed optional second and fraction of second part. + This allows parsing 2018-01-01T00+01 , 2018-01-01T00:00+01 , 2018-01-01T00:00:00+01 , 2018-01-01T00:00:00.000+01 + It won't allow parsing a timezone without an hour part as this is not allowed by iso8601 spec + + closes #49351 + +commit 9d1567b13b25ee11c94b608cadae1437729e17e2 +Author: James Rodewig +Date: Wed Jan 8 12:53:08 2020 -0600 + + [DOCS] Add overview page to analysis topic (#50515) + + Adds a 'text analysis overview' page to the analysis topic docs. + + The goals of this page are: + + * Concisely summarize the analysis process while avoiding in-depth concepts, tutorials, or API examples + * Explain why analysis is important, largely through highlighting problems with full-text searches missing analysis + * Highlight how analysis can be used to improve search results + +commit 90e66a7b9702afdeed7a4d35e7ab7f957a3e7f2c +Author: Nhat Nguyen +Date: Wed Jan 8 12:59:54 2020 -0500 + + Mute testPolicyCRUD + + Tracked at #44997 + +commit 125feecabcd65aac40a094c8897d5bf569498162 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Jan 8 19:09:14 2020 +0100 + + Guess root cause support unwrap (#50525) (#50742) + + ElasticsearchException.guessRootCauses would return wrapper exception if + inner exception was not an ElasticsearchException. Fixed to never return + wrapper exceptions. + + At least following APIs change root_cause.0.type as a result: + + _update with bad script + _index with bad pipeline + + Relates #50417 + +commit 0444da944efa16881ea59efe2c246d472a2e4750 +Author: István Zoltán Szabó +Date: Wed Jan 8 18:53:57 2020 +0100 + + [DOCS] Adds DFA resources as deleted page to redirects. (#50756) + +commit 4f2299c714c336eacf06c8f6477c05cd2707d7cd +Author: Adrien Grand +Date: Wed Jan 8 18:53:59 2020 +0100 + + Upgrade to Lucene 8.4.0. (#50518) (#50750) + +commit 615532b4f80825e315d90b013b4b310c2c539824 +Author: Lee Hinman +Date: Wed Jan 8 10:34:27 2020 -0700 + + Mute TimeSeriesLifecycleActionsIT.testHistoryIsWritten* (#50755) + + Related to #50353 + +commit f87e61ec309e62c18c0bcc2171cc32d5d797b03b +Author: James Rodewig +Date: Wed Jan 8 11:06:54 2020 -0600 + + [DOCS] Add default index-time analyzer example (#50501) + + The Analysis docs mention including a default analyzer in the index settings. However, no example snippet is included. + + This adds an example snippet that users can easily copy and adjust. + +commit f3ddd4066aaa714297572f416978b14b3d97fc35 +Author: Valentin Crettaz +Date: Wed Jan 8 17:50:03 2020 +0100 + + [DOCS] Fixed typos (_op => op) in Painless context docs (#50301) + +commit 5ff6eafb4be498be47570844c01a8c7992d88a97 +Author: blueSky1825821 <1664637427@qq.com> +Date: Thu Jan 9 00:47:11 2020 +0800 + + [Docs] Update similarity.asciidoc (#50719) + + DFRSimilarity -> DFR similarity + +commit a725896c923c220c8c39526bcf3cf0766b88a47e +Author: Armin Braun +Date: Wed Jan 8 16:33:36 2020 +0100 + + Fix and Reenable SnapshotTool Minio Tests (#50736) (#50745) + + This solves half of the problem in #46813 by moving the S3 + tests to using the shared minio fixture so we at least have + some non-3rd-party, constantly running coverage on these tests. + +commit 31158ab3d570d2d2f8a161e3c63e43c07cc05acb +Author: Adrien Grand +Date: Wed Jan 8 16:21:18 2020 +0100 + + Add per-field metadata. (#50333) + + This PR adds per-field metadata that can be set in the mappings and is later + returned by the field capabilities API. This metadata is completely opaque to + Elasticsearch but may be used by tools that index data in Elasticsearch to + communicate metadata about fields with tools that then search this data. A + typical example that has been requested in the past is the ability to attach + a unit to a numeric field. + + In order to not bloat the cluster state, Elasticsearch requires that this + metadata be small: + - keys can't be longer than 20 chars, + - values can only be numbers or strings of no more than 50 chars - no inner + arrays or objects, + - the metadata can't have more than 5 keys in total. + + Given that metadata is opaque to Elasticsearch, field capabilities don't try to + do anything smart when merging metadata about multiple indices, the union of + all field metadatas is returned. + + Here is how the meta might look like in mappings: + + ```json + { + "properties": { + "latency": { + "type": "long", + "meta": { + "unit": "ms" + } + } + } + } + ``` + + And then in the field capabilities response: + + ```json + { + "latency": { + "long": { + "searchable": true, + "aggreggatable": true, + "meta": { + "unit": [ "ms" ] + } + } + } + } + ``` + + When there are no conflicts, values are arrays of size 1, but when there are + conflicts, Elasticsearch includes all unique values in this array, without + giving ways to know which index has which metadata value: + + ```json + { + "latency": { + "long": { + "searchable": true, + "aggreggatable": true, + "meta": { + "unit": [ "ms", "ns" ] + } + } + } + } + ``` + + Closes #33267 + +commit 71054d269b48e1c45a797e9d7cd31e00e5ea416d +Author: Alexander Reelsen +Date: Wed Jan 8 14:50:43 2020 +0100 + + Sync grok patterns with logstash patterns (#50381) + + In order to ensure that logstash and Elasticsearch are able to understand + the same patterns, this commit adapts to changes in logstash, adds a few + patterns and changes a few. + +commit d3094f9d23271341837620036b500128c919f1b7 +Author: James Rodewig +Date: Wed Jan 8 07:55:21 2020 -0600 + + [DOCS] Fix typo in mapping date format docs + +commit f203c2b39d2b088a774c40caf43727ebcdfa00f5 +Author: Yannick Welsch +Date: Wed Jan 8 11:35:30 2020 +0100 + + Import replicated closed dangling indices (#50649) + + Dangling replicated closed indices are not imported properly (they miss their routing table when imported). + +commit b1ff74f6521176a5cccbc984dbc9de5f13874761 +Author: Rory Hunter +Date: Fri Nov 29 11:39:51 2019 +0000 + + New setting to prevent automatically importing dangling indices (#49174) + + Introduce a new static setting, `gateway.auto_import_dangling_indices`, which prevents dangling indices from being automatically imported. Part of #48366. + +commit 3915d4c0553d34ea33b3a2dfeac87d4e7236f7e9 +Author: Andrei Dan +Date: Wed Jan 8 11:45:26 2020 +0100 + + Make the UpdateRolloverLifecycleDateStep retryable (#50702) (#50730) + + This makes the "update-rollover-lifecycle-date" step, which is part of the + rollover action, retryable. It also adds an integration test to check the + step is retried and it eventually succeeds. + + (cherry picked from commit 5bf068522deb2b6cd2563bcf80f34fdbf459c9f2) + Signed-off-by: Andrei Dan + +commit d8c907d648ec1cb9c50d620dbe03fbbbcfb075ad +Author: Christoph Büscher +Date: Wed Jan 8 10:34:41 2020 +0100 + + Remove _reload_search_analyzer experimental status (#50696) + + Removing the experimental status in the docs and the rest specs. + +commit 293661d62ccb7995672a34e4028b161daa7112dc +Author: Tim Vernum +Date: Wed Jan 8 15:13:47 2020 +1100 + + Security should not reload files that haven't changed (#50724) + + In security we currently monitor a set of files for changes: + + - config/role_mapping.yml (or alternative configured path) + - config/roles.yml + - config/users + - config/users_roles + + This commit prevents unnecessary reloading when the file change actually doesn't change the internal structure. + + Backport of: #50207 + + Co-authored-by: Anton Shuvaev + +commit c1c0b47d5e5546ff1962894a2793eb3252d235e3 +Author: Mayya Sharipova +Date: Tue Jan 7 17:06:52 2020 -0500 + + Specify the indexname in searches (#50717) + + vector REST tests occasionally fail on 7.x because + we don't receive the expected response headers with deprecation warnings. + This happens as searchers were executed against all indices including + internal indices, whose shards did not produce expected warnings. + + This PR ensures that searchers are executed only against expected + indices. + + Closes #50716 + +commit 0b7309ec9caf1b639436a071a350661ee67dd3eb +Author: Mayya Sharipova +Date: Tue Jan 7 12:20:09 2020 -0500 + + Fix NPE bug inner_hits (#50709) + + When there several subqueries on different relations of the join field, + and only one of subqueries is using inner_hits, NPE occurs. + This PR prevents NPE error. + + Closes #50539 + +commit d0d48311f42e741d4e20d81456b68167d61ae425 +Author: Armin Braun +Date: Tue Jan 7 20:17:46 2020 +0100 + + Faster and Simpler GCS REST Mock (#50706) (#50707) + + * Faster and Simpler GCS REST Mock + + I reworked the GCS mock a little to use less copying+allocation, + log the full request body on failure to read a multi-part request + and generally be a little simpler and easy to follow to track down + the remaining issues that are causing almost daily failures from this + class's multi-part request parsing that can't be reproduced locally. + +commit de6b62f78955993bb310cb8ad9ae60b2e4a9517a +Author: James Rodewig +Date: Tue Jan 7 12:53:47 2020 -0600 + + [DOCS] Fuzzy wildcard not supported in `query_string` (#50466) + + The `query_string` does not support mixing wildcards with fuzziness. + This adds a related warning to the `query_string` docs. + +commit 1f6c1df58e2628eba50c18813716e0ea5223e4e2 +Author: Ryan Ernst +Date: Tue Jan 7 10:28:47 2020 -0800 + + Add cliSetup command to test clusters configuration (#50414) + + This commit adds a cliSetup command that can be used to run arbitrary + bin scripts to setup a test cluster. This is the same as what was + previously called setupCommands in cluster formation tasks. + + closes #50382 + +commit 20eba1e4107cd7590178b6899bec8741a3c44cf0 +Author: James Rodewig +Date: Tue Jan 7 10:54:16 2020 -0600 + + [DOCS] Reformat reverse token filter docs (#50672) + + * Updates the description and adds a Lucene link + * Adds analyze and custom analyzer snippets + +commit deb0991667815769d30cd27eaba0c8cb40c6e271 +Author: Nik Everett +Date: Tue Jan 7 11:57:41 2020 -0500 + + Teach ObjectParser a happy pattern (#50691) (#50710) + + We *very* commonly have object with ctors like: + ``` + public Foo(String name) + ``` + + And then declare a bunch of setters on the object. Every aggregation + works like this, for example. This change teaches `ObjectParser` how to + build these aggregations all on its own, without any help. This'll make + it much cleaner to parse aggs, and, probably, a bunch of other things. + It'll let us remove lots of wrapping. I've used this new power for the + `avg` aggregation just to prove that it works outside of a unit test. + +commit 8009b07ccb5f8ef2163abce2dc3bf0af6e8ca7aa +Author: James Rodewig +Date: Tue Jan 7 10:32:54 2020 -0600 + + [DOCS] Reformat truncate token filter docs (#50687) + + * Updates the description and adds a Lucene link + * Adds analyze, custom analyzer, and custom filter snippets + * Adds parameter documentation + +commit d5f4790f90b9590b6168eb9c67851e0937245ca7 +Author: arkel-s <54832605+arkel-s@users.noreply.github.com> +Date: Tue Jan 7 17:07:29 2020 +0100 + + [DOCS] Add example format for `date_optional_time` (#50458) + + Adds an example format for `date_optional_time` to the `format` mapping + parameter docs. + + Closes #50457 + +commit c3d207f437f3f0d1d10b7946434dd95905aa13e6 +Author: Nhat Nguyen +Date: Tue Jan 7 10:08:13 2020 -0500 + + Disable auto refresh in testSegmentsStats (#50689) + + If an auto-refresh happens, then version_map_memory is reset to 0. By + default, the auto-refresh occurs for every second in the first 30 + seconds until search becomes idle. + + Closes #50362 + +commit 0753915eedde39c791967a65189aa2d95f06409f +Author: James Rodewig +Date: Tue Jan 7 09:17:54 2020 -0600 + + [DOCS] Update SQL REST API pages for new structure (#50690) + + #43007 restructured the SQL REST API docs so they display across several pages. + + This updates up a reference that assumes a single page in the "Paginating through a large response" section. It also reformats a tip for the Kibana console. + + Closes #50688 + +commit 060e0a6277d675b9e8623685c60fa872d8b10b3c +Author: Benjamin Trent +Date: Tue Jan 7 09:21:59 2020 -0500 + + [ML][Inference] Add support for models shipped as resources (#50680) (#50700) + + This adds support for models that are shipped as resources in the ML plugin. The first of which is the `lang_ident` model. + +commit 98ca9500e85a0637a74c2ee9595f174732d5fed5 +Author: Hendrik Muhs +Date: Tue Jan 7 13:51:51 2020 +0100 + + implement a workaround for remote cluster validation (#50460) + + In 7.x an internal API used for validating remote cluster does not throw, see #50420 for the + details. This change implements a workaround for remote cluster validation, only for 7.x branches. + + fixes #50420 + +commit 4116452d90c635b54fdc0526474a35166927d8c7 +Author: Przemysław Witek +Date: Tue Jan 7 13:41:37 2020 +0100 + + Implement testStopAndRestart for ClassificationIT (#50585) (#50698) + +commit 35453e2b0e770fe155566cc628e6a2caab87f73a +Author: David Roberts +Date: Tue Jan 7 10:23:40 2020 +0000 + + [ML] Improve uniqueness of result document IDs (#50644) + + Switch from a 32 bit Java hash to a 128 bit Murmur hash for + creating document IDs from by/over/partition field values. + The 32 bit Java hash was not sufficiently unique, and could + produce identical numbers for relatively common combinations + of by/partition field values such as L018/128 and L017/228. + + Fixes #50613 + +commit 46d600c4469c5b82ea8ec2a8134379d6f539490f +Author: David Roberts +Date: Tue Jan 7 10:11:09 2020 +0000 + + [ML] Fix off-by-one error in ml_classic tokenizer end offset (#50655) + + The end offset of a tokenizer is supposed to point one past the + end of the input, not to the end character of the input. The + ml_classic tokenizer was erroneously doing the latter. + +commit 2039cc813be0864ea748f64202841133058fdbc3 +Author: David Turner +Date: Tue Jan 7 09:02:35 2020 +0000 + + Fix testDelayVariabilityAppliesToFutureTasks (#50667) + + This test seems to be bogus as it was confusing a nominal execution time with a + delay (i.e. an elapsed time). This commit reworks the test to address this. + + Fixes #50650 + +commit a3ab7eb95dc2f1772f4d48a87e9f3156d4e33985 +Author: Alan Woodward +Date: Tue Jan 7 09:31:22 2020 +0000 + + Correctly handle MSM for nested disjunctions (#50669) + + With the rewrite of the percolator's QueryAnalyzer to use lucene's QueryVisitor API, + term queries that are direct children of a boolean query are handled separately from + other children. This works fine for conjunctions, but for disjunctions we need to + treat the extracted terms from these direct descendents along with extractions from + more deeply nested children to ensure that minimum-should-match requirements + are met correctly. + + This commit changes the logic in QueryAnalyzer#getResult() to bundle child term + results with all other results before handling them. + + Fixes #50305 + +commit a2ef0e88307e4bde5b81c8df2a7914ea5a05183f +Author: Yannick Welsch +Date: Tue Jan 7 09:38:50 2020 +0100 + + Check allocation id when failing shard on recovery (#50656) + + A failure of a recovering shard can race with a new allocation of the shard, and cause the new + allocation to be failed as well. This can result in a shard being marked as initializing in the cluster + state, but not exist on the node anymore. + + Closes #50508 + +commit 552edd862e93af9595ddc3bd7bdbc33dd3b1518a +Author: Lee Hinman +Date: Mon Jan 6 15:24:24 2020 -0700 + + [7.x] Add aditional logging for ILM history store tests (#5062… (#50678) + + * Add aditional logging for ILM history store tests (#50624) + + These tests use the same index name, making it hard to read logs when + diagnosing the failures. Additionally more information about the current + state of the index could be retrieved when failing. + + This changes these two things in the hope of capturing more data about + why this fails on some CI nodes but not others. + + Relates to #50353 + +commit e5191e77e37b1cfc542b89045e0a1f954278de39 +Author: Jay Modi +Date: Mon Jan 6 14:57:10 2020 -0700 + + Remove unused IndicesOptions#fromByte method (#50683) + + This change removes a no longer used method, `fromByte`, in + IndicesOptions. This method was necessary for backwards compatibility + with versions prior to 6.4.0 and was used when talking to those + versions. However, the minimum wire compatibility version has changed + and we no longer use this code. + + Backport of #50665 + +commit 7fd84a03a0fca7f61901bf557ab47c79deba4030 +Author: Nik Everett +Date: Mon Jan 6 16:34:07 2020 -0500 + + Drop references to deprecated logger (#50474) (#50681) + + This drops all remaining references to `BaseRestHandler.logger` which + has been deprecated for something like a year now. I replaced all of the + references with locally declared loggers which is so much less spooky + action at a distance to me. + +commit 06cea5136eeb15cc5ff314849a9c344d63b7e674 +Author: Benjamin Trent +Date: Mon Jan 6 16:26:29 2020 -0500 + + [ML] construct new random generator on each persistence call (#50657) (#50684) + + Sharing a random generator may cause test failures as non-threadsafe random generators are periodically utilized in tests (see: https://github.com/elastic/elasticsearch/issues/50651) + + This change constructs a calls `Randomness.get()` within the `bulkIndexWithRetry` method so that the returned `Random` object is only used in a single thread. Before, the member variable could have been used between threads, which caused test failures. + +commit 5ab9e75e283b193f9e059a74733de7de9a6c2d5f +Author: Benjamin Trent +Date: Mon Jan 6 16:24:03 2020 -0500 + + [7.x] [ML][Inference] lang_ident model (#50292) (#50675) + + * [ML][Inference] lang_ident model (#50292) + + This PR contains a java port of Google's CLD3 compact NN model https://github.com/google/cld3 + + The ported model is formatted to fit within our inference model formatting and stored as a resource in the `:xpack:ml:` plugin and is under basic license. + + The model is broken up into two major parts: + - Preprocessing through the custom embedding (based on CLD3's embedding layer) + - Pushing the embedded text through the two layers of fully connected shallow NN. + + Main differences between this port and CLD3: + - We take advantage of Java's internal Unicode handling where possible (i.e. codepoints, characters, decoders, etc.) + - We do not trim down input text by removing duplicated tokens + - We do not encode doubles/floats as longs/integers. + +commit 76bb661023fc4ab37623fe32303fd2d7dc7a4c3b +Author: Nik Everett +Date: Mon Jan 6 14:32:03 2020 -0500 + + Replace AggParseContext with a String (backport of #50625) (#50679) + + We used to have a *ton* off stuff in the `AggParseContext` but now we + parse aggs entirely with named xcontent. So we don't need the context + any more. + +commit f52af7977d475279e10127b3bc9b5f7610b3bb82 +Author: Benjamin Trent +Date: Mon Jan 6 14:05:04 2020 -0500 + + [ML][Inference] minor cleanup for inference (#50444) (#50676) + +commit 1b28af489ff6a8a371dcf0437f862767c5210d6d +Author: Nik Everett +Date: Mon Jan 6 14:03:30 2020 -0500 + + Fix bare warnings on RollupJobTests (#50633) (#50677) + + Silences some ugly warnings. + +commit 926c0aa74c0d621e624bde8dfc5a48a05d5ef16b +Author: Nhat Nguyen +Date: Mon Jan 6 13:30:08 2020 -0500 + + Fix testRelocationEstablishedPeerRecoveryRetentionLeases (#50673) + + The redNodes are calculated incorrectly. + + Closes #50660 + +commit 72a405fafb472a5f3f4d3fed301cc9a78f8f41d5 +Author: Armin Braun +Date: Mon Jan 6 19:27:57 2020 +0100 + + Fix GCS Mock Broken Handling of some Blobs (#50666) (#50671) + + * Fix GCS Mock Broken Handling of some Blobs + + We were incorrectly handling blobs starting in `\r\n` which broke + tests randomly when blobs started on these. + + Relates #49429 + +commit f576aefd0f362233c097fe7c2844c5918f3b0f5d +Author: Nik Everett +Date: Mon Jan 6 12:57:43 2020 -0500 + + Replace bespoke parser for significance heuristics (#50623) (#50659) + + This replaces the hand written xcontent parsers for significance + heristics with `ObjectParser` and parsing named xcontent. + + As a happy accident, this was the last user of `ParseFieldRegistry` so + this PR entirely removes that class. + + Closes #25519 + +commit fa57813c6d9bc1734aec79b68bec1641b3df82b1 +Author: Tim Brooks +Date: Mon Jan 6 11:12:00 2020 -0600 + + Remove races in ProxyConnectionStrategyTests (#50620) + + Currently, we use delayed address resolution in the proxy strategy tests + to allow tests to connect to different addresses. Unfortunately, this + has the potential to introduce races as the address is resolved each + connection attempt. The number of connection attempts can vary based on + when connections are opening and closing. This commit modifies the test + be allowing them to specifically control which address is used. + + Related to #50618 + +commit 074866256bc6d15122498de3e8529121f73e55e9 +Author: James Rodewig +Date: Mon Jan 6 08:38:21 2020 -0600 + + [DOCS] Remove unneeded redirects (#50510) + + The docs/reference/redirects.asciidoc file stores a list of relocated or + deleted pages for the Elasticsearch Reference documentation. + + This prunes several older redirects that are no longer needed. + +commit 7be43e9f6d0e9dcf653519099596148910cbdb52 +Author: Martijn van Groningen +Date: Mon Jan 6 15:37:02 2020 +0100 + + Fix ingest stats test bug. (#50653) + + This test code fixes a serialization test bug: + https://gradle-enterprise.elastic.co/s/7x2ct6yywkw3o + + Rarely stats for the same processor are generated and + the production code then sums up these stats. However + the test code wasn't summing up in that case, + which caused inconsistencies between the actual and expected results. + + Closes #50507 + +commit 1299dda437a2480cdf4c83931bf850ffa99fe389 +Author: James Rodewig +Date: Mon Jan 6 07:45:49 2020 -0600 + + [DOCS] Warn about using `geo_centroid` as sub-agg to `geohash_grid` (#50038) + + If `geo_point fields` are multi-valued, using `geo_centroid` as a + sub-agg to `geohash_grid` could result in centroids outside of bucket + boundaries. + + This adds a related warning to the geo_centroid agg docs. + +commit b71490b06bdd7c9c9df51ac81bfbeea9347ae469 +Author: Nhat Nguyen +Date: Mon Jan 6 08:44:30 2020 -0500 + + Deprecate indices without soft-deletes (#50502) (#50634) + + Soft-deletes will be enabled for all indices in 8.0. Hence, we should + deprecate new indices without soft-deletes in 7.x. + + Backport of #50502 + +commit ec0ec61881bf70fc35f3540ec16f3c81ae4762f8 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jan 6 13:24:07 2020 +0100 + + Deleted docs disregarded for if_seq_no check (#50526) + + Previously, as long as a deleted version value was kept as a tombstone, + another index or delete operation against the same id would leak that + the doc had existed (through seq_no info) or would allow the operation + if the client forged the seq_no. Fixed to disregard info on deleted docs + when doing seq_no based optimistic concurrency check. + +commit 5533e1172c4ac4f5e510109ebb1f5047452896bf +Author: Nikita Glashenko +Date: Mon Jan 6 15:46:08 2020 +0400 + + Add tests for remaining IntervalsSourceProvider implementations (#50326) + + This PR adds unit tests for wire and xContent serialization of remaining IntervalsSourceProvider + implementations. + + Closes #50150 + +commit 66c690922c36c720f79e129d918387b692b07fbc +Author: David Turner +Date: Mon Jan 6 11:07:35 2020 +0000 + + Collect shard sizes for closed indices (#50645) + + Today the `InternalClusterInfoService` collects information on the sizes of + shards of open indices, but does not consider closed indices. This means that + shards of closed indices are treated as having zero size when they are being + allocated. This commit fixes this, obtaining the sizes of all shards. + + Relates #33888 + +commit 312bf44601efe7de7d7862b68dd663b878dbdbc8 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jan 6 12:18:12 2020 +0100 + + Workaround for JDK 14 EA FileChannel.map issue (#50523) + + FileChannel.map provokes static initialization of ExtendedMapMode in + JDK14 EA, which needs elevated privileges. + + Relates #50512 + +commit 9ae3cd2a78bc1fd1a2728910f2408965279c313a +Author: Albert Zaharovits +Date: Mon Jan 6 13:15:55 2020 +0200 + + Add 'monitor_snapshot' cluster privilege (#50489) (#50647) + + This adds a new cluster privilege `monitor_snapshot` which is a restricted + version of `create_snapshot`, granting the same privileges to view + snapshot and repository info and status but not granting the actual + privilege to create a snapshot. + + Co-authored-by: j-bean + +commit 0f2d26bdcae370a1225e58e06585907c5c4082eb +Author: Martijn van Groningen +Date: Mon Dec 23 10:48:58 2019 +0100 + + Unmute 'Test url escaping with url mustache function' webhook watcher test (#50439) + + Some changes had to be made in order to make the test pass due to the removal or types. + Added some more assertions. The failure description in this comment [0] indicates that the rest handler couldn't be found. The test passes now. + I plan to merge this into master and see how CI reacts, if it handles this change well then I will also unmute this test in 7 dot x branch. + + Also check watch count after stopping watcher in test teardown and + disabled slm in smoke test watcher qa test. + + Relates to #41172 + + 0: https://github.com/elastic/elasticsearch/issues/41172#issuecomment-496993976 + +commit 2362c430cdc0c3754a85ede9b0708a654ccd55c5 +Author: Nik Everett +Date: Sun Jan 5 16:20:38 2020 -0500 + + Clean up wire test case a bit (#50627) (#50632) + + * Adds JavaDoc to `AbstractWireTestCase` and + `AbstractWireSerializingTestCase` so it is more obvious you should prefer + the latter if you have a choice + * Moves the `instanceReader` method out of `AbstractWireTestCase` becaue + it is no longer used. + * Marks a bunch of methods final so it is more obvious which classes are + for what. + * Cleans up the side effects of the above. + +commit 8819fa4ebef6bfe6f5f9685f3c57e9ba5d8445ca +Author: Armin Braun +Date: Fri Jan 3 21:32:36 2020 +0100 + + Make EC2 Discovery Cache Empty Seed Hosts List (#50607) (#50626) + + Follow up to #50550. Cache empty nodes lists (`fetchDynamicNodes` will return an empty list in case of failure) + now that the plugin properly retries requests to AWS EC2 APIs. + +commit a45de8a96b5f44b60b2f30e49e8d129126347958 +Author: Nik Everett +Date: Fri Jan 3 14:47:42 2020 -0500 + + x-content: Support collapsed named objects (#50564) (#50619) + + This adds support for "collapsed" named object to `ObjectParser`. In + particular, this supports the sort of xcontent that we use to specify + significance heuristics. See #25519 and this example: + + ``` + GET /_search + { + "query" : { + "terms" : {"force" : [ "British Transport Police" ]} + }, + "aggregations" : { + "significant_crime_types" : { + "significant_terms" : { + "field" : "crime_type", + "mutual_information" : { <<------- This is the name + "include_negatives": true + } + } + } + } + } + ``` + + I believe there are a couple of things that work this way. + + I've held off on moving the actual parsing of the significant heuristics + to this code to keep the review more compact. The moving is pretty + mechanical stuff in the aggs framework. + +commit 45663ac1a8c2bef8555b08f61644ce43debc9cd7 +Author: Nik Everett +Date: Fri Jan 3 13:28:55 2020 -0500 + + Use Void context on parsers where possible (#50573) (#50617) + + *Most* of our parsing can be done without passing any extra context into + the parser that isn't already part of the xcontent stream. While I was + looking around at the places that *do* need a context I found a few + places that were declared to need a context but don't actually need it. + +commit 6c8868e95538a17cf5a49ce089073417430b2564 +Author: Christoph Büscher +Date: Fri Jan 3 18:25:02 2020 +0100 + + Mute TimeSeriesLifecycleActionsIT.testHistoryIsWrittenWithSuccess + + Also muting TimeSeriesLifecycleActionsIT.testHistoryIsWrittenWithFailure. + + Tracked in #50353 + +commit 62969c35cd5ffae7a6ed76c928acbabf5c192d13 +Author: Lisa Cawley +Date: Fri Jan 3 09:07:08 2020 -0800 + + [DOCS] Adds missing timing_stats descriptions (#50574) + +commit 4d58656065af81b1e333f17347c150b4309d299f +Author: Nik Everett +Date: Fri Jan 3 11:48:11 2020 -0500 + + Declare remaining parsers `final` (#50571) (#50615) + + We have about 800 `ObjectParsers` in Elasticsearch, about 700 of which + are final. This is *probably* the right way to declare them because in + practice we never mutate them after they are built. And we certainly + don't change the static reference. Anyway, this adds `final` to these + parsers. + + I found the non-final parsers with this: + ``` + diff \ + <(find . -type f -name '*.java' -exec grep -iHe 'static.*PARSER\s*=' {} \+ | sort) \ + <(find . -type f -name '*.java' -exec grep -iHe 'static.*final.*PARSER\s*=' {} \+ | sort) \ + 2>&1 | grep '^<' + ``` + +commit 856607b5a68d1c7150c41089348997bef56e24d3 +Author: Andrei Dan +Date: Fri Jan 3 18:04:26 2020 +0200 + + Guard against null geoBoundingBox (#50506) (#50608) + + A geo box with a top value of Double.NEGATIVE_INFINITY will yield an empty + xContent which translates to a null `geoBoundingBox`. This commit marks the + field as `Nullable` and guards against null when retrieving the `topLeft` + and `bottomRight` fields. + + Fixes https://github.com/elastic/elasticsearch/issues/50505 + + (cherry picked from commit 051718f9b1e1ca957229b01e80d7b79d7e727e14) + Signed-off-by: Andrei Dan + +commit 1abecad21be5dcdcbcc252eb93f78350646b9581 +Author: Nik Everett +Date: Fri Jan 3 10:58:15 2020 -0500 + + Mark some constants in decay functions final (#50569) (#50575) + + This marks a couple of constants in the `DecayFunctionBuilder` as final. + They are written in CONSTANT_CASE and used as constants but not final + which is a little confusing and might lead to sneaky bugs. + +commit 44827e577e7ad0b591fe373737a7a210e619849d +Author: Orhan Toy +Date: Fri Jan 3 16:15:18 2020 +0100 + + [DOCS] Fix missing quote in script-score-query.asciidoc (#50590) + +commit 218bd19034d672201379ea351087c3876b667bc1 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jan 3 15:28:47 2020 +0100 + + Improve FutureUtils.get exception handling (#50339) (#50417) + + FutureUtils.get() would unwrap ElasticsearchWrapperExceptions. This + is trappy, since nearly all usages of FutureUtils.get() expected only to + not have to deal with checked exceptions. + + In particular, StepListener builds upon ListenableFuture which uses + FutureUtils.get to be informed about the exception passed to onFailure. + This had the bad consequence of masking away any exception that was an + ElasticsearchWrapperException like RemoteTransportException. + Specifically for recovery, this made CircuitBreakerExceptions happening + on the target node look like they originated from the source node. + + The only usage that expected that behaviour was AdapterActionFuture. + The unwrap behaviour has been moved to that class. + +commit 3c971f291181698753344fb8750ae95e6e49bd65 +Author: Andrei Dan +Date: Fri Jan 3 16:19:58 2020 +0200 + + ILM retryable async action steps (#50522) (#50591) + + This adds support for retrying AsyncActionSteps by triggering the async + step after ILM was moved back on the failed step (the async step we'll + be attempting to run after the cluster state reflects ILM being moved + back on the failed step). + + This also marks the RolloverStep as retryable and adds an integration + test where the RolloverStep is failing to execute as the rolled over + index already exists to test that the async action RolloverStep is + retried until the rolled over index is deleted. + + (cherry picked from commit 8bee5f4cb58a1242cc2ef4bc0317dae6c8be49d3) + Signed-off-by: Andrei Dan + +commit 0bcbddecf8db412c72174f9e7f91aa9f9bfdebba +Author: István Zoltán Szabó +Date: Fri Jan 3 14:49:43 2020 +0100 + + [DOCS] Fine-tunes training_percent definition. (#50601) + +commit e6a469cc7410c2ba13a59753198bca6c5d848ff7 +Author: James Rodewig +Date: Fri Jan 3 08:34:11 2020 -0500 + + [DOCS] Reformat uppercase token filter docs (#50555) + + * Updates the description and adds a Lucene link + * Adds analyze and custom analyzer snippets + +commit ca0828ba072a8f2f87955eeebf89579015f51259 +Author: Dimitris Athanasiou +Date: Fri Jan 3 13:46:02 2020 +0200 + + [7.x][ML] Implement force deleting a data frame analytics job (#50553) (#50589) + + Adds a `force` parameter to the delete data frame analytics + request. When `force` is `true`, the action force-stops the + jobs and then proceeds to the deletion. This can be used in + order to delete a non-stopped job with a single request. + + Closes #48124 + + Backport of #50553 + +commit 5655d6a1c1ba0c8d59c4756d9d8e9d13bca350fd +Author: kkewwei +Date: Fri Jan 3 19:23:20 2020 +0800 + + Log index name when updating index settings (#49969) + + Today we log changes to index settings like this: + + updating [index.setting.blah] from [A] to [B] + + The identity of the index whose settings were updated is conspicuously absent + from this message. This commit addresses this by adding the index name to these + messages. + + Fixes #49818. + +commit d466efbff6dc24d3d3b5e23148b2f7afe9f9c7e8 +Author: Alan Woodward +Date: Fri Jan 3 10:41:30 2020 +0000 + + Fix fuzzy interval invocation in rest test + +commit 8b362c657bab82333cce1d4a376114a17f55d3a8 +Author: Alan Woodward +Date: Fri Jan 3 09:55:53 2020 +0000 + + Add fuzzy intervals source (#49762) + + This intervals source will return terms that are similar to an input term, up to + an edit distance defined by fuzziness, similar to FuzzyQuery. + + Closes #49595 + +commit a34b3f133c49158d084bc4ed71f2d638c9f202fc +Author: István Zoltán Szabó +Date: Fri Jan 3 10:41:38 2020 +0100 + + [DOCS] Specifies the possible data types of classification dependent_variable (#50582) + +commit 8917c05df8eb32298f193314b219456f1b3fa08a +Author: Przemysław Witek +Date: Fri Jan 3 10:23:51 2020 +0100 + + [7.x] Synchronize processInStream.close() call (#50581) + +commit 0d78aa2708ac1299819756d66a5f7242831f6466 +Author: Lee Hinman +Date: Thu Jan 2 16:57:51 2020 -0700 + + Don't dump a stacktrace for invalid patterns when executing elasticsearch-croneval (#49744) (#50578) + + Co-authored-by: bellengao + +commit 81a9cff16f525c0f81478fefde876f08c981244d +Author: Lisa Cawley +Date: Thu Jan 2 11:23:26 2020 -0800 + + [7.x][DOCS] Remove redundant results from ML APIs (#50565) + +commit e19585b47f2d8575622c51ce93ce1ad2482c4781 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 2 19:19:52 2020 +0100 + + Enhance TransportReplicationAction assertions (#49081) + + Include failure into assertion error when replication action discovers + that it has been double triggered. + +commit 338dd642c4418ccc89b960d10dc4280c0c32cbe1 +Author: James Rodewig +Date: Thu Jan 2 13:13:00 2020 -0500 + + [DOCS] Correct typos in Painless datetime docs (#50563) + + Fixes several typos and grammar errors raised by @glenacota in #47512. + + Co-authored-by: Guido Lena Cota + +commit b209e7b6ff8f00b8062aa8276c39f5acea302c61 +Author: David Turner +Date: Thu Jan 2 17:19:42 2020 +0000 + + Remove included docs (#50557) + + In #50499 we accidentally duplicated the docs for the `?local` parameter to the + `GET _cat/nodes` API. This commit removes the duplicate docs. + +commit 8092a4991e4305eb8162077bfd5333669e680c3e +Author: Armin Braun +Date: Thu Jan 2 17:39:59 2020 +0100 + + Make EC2 Discovery Plugin Retry Requests (#50550) (#50558) + + Use the default retry condition instead of never retrying in the discovery plugin causing hot retries upstream and add a test that verifies retrying works. + + Closes #50462 + +commit b36a8ab141d4095a93f1b036ae1410a0667f5b94 +Author: Nik Everett +Date: Thu Jan 2 10:47:38 2020 -0500 + + Make some ObjectParsers final (#50471) (#50556) + + We have about 800 `ObjectParsers` in Elasticsearch, about 700 of which + are final. This is *probably* the right way to declare them because in + practice we never mutate them after they are built. And we certainly + don't change the static reference. Anyway, this adds `final` to a bunch + of these parsers, mostly the ones in xpack and their "paired" parsers in + the high level rest client. I picked these just to have somewhere to + break the up the change so it wouldn't be huge. + + I found the non-final parsers with this: + ``` + diff \ + <(find . -type f -name '*.java' -exec grep -iHe 'static.*PARSER\s*=' {} \+ | sort) \ + <(find . -type f -name '*.java' -exec grep -iHe 'static.*final.*PARSER\s*=' {} \+ | sort) \ + 2>&1 | grep '^<' + ``` + +commit 55107ce8ae46f96a51d28681f87b3c6986be0a8d +Author: Nik Everett +Date: Thu Jan 2 10:02:55 2020 -0500 + + Docs: Refine note about `after_key` (#50475) + + * Docs: Refine note about `after_key` + + I was curious about composite aggregations, specifically I wanted to + know how to write a composite aggregation that had all of its buckets + filtered out so you *had* to use the `after_key`. Then I saw that we've + declared composite aggregations not to work with pipelines in #44180. So + I'm not sure you *can* do that any more. Which makes the note about + `after_key` inaccurate. This rejiggers that section of the docs a little + so it is more obvious that you send the `after_key` back to us. And so + it is more obvious that you should *only* use the `after_key` that we + give you rather than try to work it out for yourself. + + * Apply suggestions from code review + + Co-Authored-By: James Rodewig + + Co-authored-by: James Rodewig + +commit 7539fbb30fc4069c8ecc26b0e60d85625022467c +Author: Oleg <25862932+olegbonar@users.noreply.github.com> +Date: Thu Jan 2 18:22:40 2020 +0400 + + Deprecate the 'local' parameter of /_cat/nodes (#50499) + + The cat nodes API performs a `ClusterStateAction` then a `NodesInfoAction`. + Today it accepts the `?local` parameter and passes this to the + `ClusterStateAction` but this parameter has no effect on the `NodesInfoAction`. + This is surprising, because `GET _cat/nodes?local` looks like it might be a + completely local call but in fact it still depends on every node in the + cluster. + + This commit deprecates the `?local` parameter on this API so that it can be + removed in 8.0. + + Relates #50088 + +commit 4ecabe496fd887d181a4212ab4382580f83c1b1b +Author: Przemysław Witek +Date: Thu Jan 2 15:28:20 2020 +0100 + + Mute testStopAndRestart test case (#50551) + +commit 1599af8428ebf247f54cecd6be4f3af4f354428c +Author: Christoph Büscher +Date: Thu Jan 2 14:28:50 2020 +0100 + + Fix type conversion problem in Eclipse (#50549) + + Eclipse 4.13 shows a type mismatch error in the affected line because it cannot + correctly infer the boolean return type for the method call. Assigning return + value to a local variable resolves this problem. + +commit ab5a69d1e27ab5931333c797aa02752ea8ec5ee9 +Author: Lisa Cawley +Date: Tue Dec 31 13:21:17 2019 -0800 + + [7.x][DOCS] Move machine learning results definitions into APIs (#50543) + +commit f8eef43fc6cfb28eb79226a33a069e11801db7ad +Author: Lisa Cawley +Date: Tue Dec 31 10:53:05 2019 -0800 + + [7.x][DOCS] Move model snapshot resource definitions into APIs (#50540) + +commit 3fb4f1b5bf4f954e4bb1defa604c6d7596ee54f3 +Author: Lisa Cawley +Date: Mon Dec 30 14:55:36 2019 -0800 + + [DOCS] Moves job count resource definitions into API (#50529) + +commit ecf295fa773407d376694b8ac383a617b57b7df5 +Author: Jake +Date: Mon Dec 30 15:38:59 2019 -0500 + + [DOCS] Bump copyright to 2019 for Java HLRC license (#50206) + +commit 9fdfb075bb1952859519853cded3a9fec3a74588 +Author: Gilad Gal +Date: Mon Dec 30 21:25:15 2019 +0200 + + Deleted 'a' before plural 'messages' + + Deleted 'a' before plural 'messages' + +commit 4b829db593c6e7818b79afdb4a7f0de733ab955f +Author: Lisa Cawley +Date: Mon Dec 30 09:35:16 2019 -0800 + + [7.x][DOCS] Move datafeed resource definitions into APIs (#50516) + +commit 8869f2b9b2ea8fe159c4511e3f88483d9b32e1cb +Author: Lisa Cawley +Date: Mon Dec 30 07:03:38 2019 -0800 + + [DOCS] Adds intro for OIDC realm (#50485) + +commit e7d5443bf550a62c9a775062ca31ad397b83e9c5 +Author: riverbuilding +Date: Mon Dec 30 08:48:10 2019 -0500 + + [DOCS] Correct Painless operator typos (#50472) + +commit cad0f6bf281d99a85047580cd0338b307bf8b77f +Author: Tim Vernum +Date: Mon Dec 30 14:42:32 2019 +1100 + + Do not load SSLService in plugin contructor (#50519) + + XPackPlugin created an SSLService within the plugin contructor. + This has 2 negative consequences: + + 1. The service may be constructed based on a partial view of settings. + Other plugins are free to add setting values via the + additionalSettings() method, but this (necessarily) happens after + plugins have been constructed. + + 2. Any exceptions thrown during the plugin construction are handled + differently than exceptions thrown during "createComponents". + Since SSL configurations exceptions are relatively common, it is + far preferable for them to be thrown and handled as part of the + createComponents flow. + + This commit moves the creation of the SSLService to + XPackPlugin.createComponents, and alters the sequence of some other + steps to accommodate this change. + + Backport of: #49667 + +commit 72840c0cb2b7d67f03c9d8e9a5bdfae287d88fbc +Author: Lisa Cawley +Date: Fri Dec 27 13:30:26 2019 -0800 + + [7.x][DOCS] Move anomaly detection job resource definitions into APIs (#50490) + +commit 7a14607a250095391762d2adc50ba715b41d0a30 +Author: James Rodewig +Date: Fri Dec 27 11:00:51 2019 -0500 + + [DOCS] Abbreviate token filter titles (#50511) + +commit 3f7f31b6b00957c5e65f2b100c1b8fd9860baa4c +Author: James Rodewig +Date: Thu Dec 26 14:20:51 2019 -0500 + + [DOCS] Fix search request body links (#50500) + + PR #44238 changed several links related to the Elasticsearch search request body API. This updates several places still using outdated links or anchors. + + This will ultimately let us remove some redirects related to those link changes. + +commit e7c15a5c6eb2f9ac9578f85d004c58e772f25cce +Author: Nhat Nguyen +Date: Thu Dec 26 08:58:34 2019 -0500 + + Ensure relocating shards establish peer recovery retention leases (#50486) + + We forgot to establish peer recovery retention leases for relocating primaries + without soft-deletes. + + Relates #50351 + +commit 261566154be0cc9b124a649b38bca6b9b0c277b9 +Author: James Rodewig +Date: Thu Dec 26 12:32:27 2019 -0500 + + [DOCS] Fix search request body link (#50498) + +commit 7713221733ddf229e84a878f880b6eb9609f3461 +Author: Nhat Nguyen +Date: Thu Dec 26 08:56:00 2019 -0500 + + Fix testCancelRecoveryDuringPhase1 (#50449) + + testCancelRecoveryDuringPhase1 uses a mock of IndexShard, which can't + create retention leases. We need to stub method createRetentionLease. + + Relates #50351 + Closes #50424 + +commit ef467cc6f5807855b12ff8756b1e07b8cbdf0910 +Author: James Rodewig +Date: Thu Dec 26 07:49:41 2019 -0500 + + [DOCS] Remove unneeded redirects (#50476) + + The docs/reference/redirects.asciidoc file stores a list of relocated or + deleted pages for the Elasticsearch Reference documentation. + + This prunes several older redirects that are no longer needed and + don't require work to fix broken links in other repositories. + +commit 1cec87c0e6bad7c3226ff664af2f6d5be4d89282 +Author: James Rodewig +Date: Thu Dec 26 07:41:23 2019 -0500 + + [DOCS] Document `transport` and `http` node stats (#50473) + + Documents the `transport` and `http` parameters returned by the + `_nodes/stats` API. + +commit d479e0563a6794579c9e6c408c6fc6a475f0f58c +Author: Lisa Cawley +Date: Tue Dec 24 10:22:05 2019 -0800 + + [7.x][DOCS] Augments ML shared definitions (#50487) + +commit f57569bf5cb387e8ca7cf8fdc8f7b3437c79bbdf +Author: Yannick Welsch +Date: Fri Dec 20 11:59:09 2019 +0100 + + Mute RecoverySourceHandlerTests.testCancelRecoveryDuringPhase1 + + Relates #50424 + +commit 10ed1ae1d2ff6785fa1df07e4269a4423d842c02 +Author: Martijn van Groningen +Date: Tue Dec 24 15:11:58 2019 +0100 + + Add remote info to the HLRC (#50483) + + The additional change to the original PR (#49657), is that `org.elasticsearch.client.cluster.RemoteConnectionInfo` now parses the initial_connect_timeout field as a string instead of a TimeValue instance. + + The reason that this is needed is because that the initial_connect_timeout field in the remote connection api is serialized for human consumption, but not for parsing purposes. + Therefore the HLRC can't parse it correctly (which caused test failures in CI, but not in the PR CI + :( ). The way this field is serialized needs to be changed in the remote connection api, but that is a breaking change. We should wait making this change until rest api versioning is introduced. + + Co-Authored-By: j-bean + + Co-authored-by: j-bean + +commit 33204c2055d03f222e72013763b3558baebc5823 +Author: Nhat Nguyen +Date: Fri Dec 20 00:39:53 2019 -0500 + + Use peer recovery retention leases for indices without soft-deletes (#50351) + + Today, the replica allocator uses peer recovery retention leases to + select the best-matched copies when allocating replicas of indices with + soft-deletes. We can employ this mechanism for indices without + soft-deletes because the retaining sequence number of a PRRL is the + persisted global checkpoint (plus one) of that copy. If the primary and + replica have the same retaining sequence number, then we should be able + to perform a noop recovery. The reason is that we must be retaining + translog up to the local checkpoint of the safe commit, which is at most + the global checkpoint of either copy). The only limitation is that we + might not cancel ongoing file-based recoveries with PRRLs for noop + recoveries. We can't make the translog retention policy comply with + PRRLs. We also have this problem with soft-deletes if a PRRL is about to + expire. + + Relates #45136 + Relates #46959 + +commit 1dc98ad61762599befc4114653b018240f5cea7c +Author: Nhat Nguyen +Date: Mon Dec 23 11:55:49 2019 -0500 + + Ensure global checkpoint was advanced and synced + + We need to make sure that the global checkpoints and peer recovery + retention leases were advanced to the max_seq_no and synced; otherwise, + we can risk expiring some peer recovery retention leases because of the + file-based recovery threshold. + + Relates #49448 + +commit 6a3d1a077e8e2158be9639cc30e97dda0101e6c3 +Author: Orhan Toy +Date: Mon Dec 23 20:38:37 2019 +0100 + + [DOCS] Fixes "enables you to" typos (#50225) + +commit bed121efaf2be6e84ccc3687bc30796411af0a6a +Author: Tal Levy +Date: Mon Dec 23 11:21:39 2019 -0800 + + [7.x-backport] Centralize BoundingBox logic to a dedicated class (#50469) + + Both geo_bounding_box query and geo_bounds aggregation have + a very similar definition of a "bounding box". A lot of this + logic (serialization, xcontent-parsing, etc) can be centralized + instead of having separated efforts to do the same things + +commit 694b119f0a28a9af449997e6714ac293a5913a84 +Author: James Rodewig +Date: Mon Dec 23 13:11:31 2019 -0500 + + [DOCS] Percentile aggs are non-deterministic (#50468) + + Percentile aggregations are non-deterministic. A percentile aggregation + can produce different results even when using the same data. + + Based on [this discuss post][0], the non-deterministic property stems + from processes in Lucene that can affect the order in which docs are + provided to the aggregation. + + This adds a warning stating that the aggregation is non-deterministic + and what that means. + + [0]: https://discuss.elastic.co/t/different-results-for-same-query/111757 + +commit 01293ebad54080bcdb82d4570f6f8b31b1a2a788 +Author: Nik Everett +Date: Mon Dec 23 12:38:17 2019 -0500 + + Fix docs typos (#50365) (#50464) + + Fixes a few typos in the docs. + + Co-authored-by: Xiang Dai <764524258@qq.com> + +commit d5cec7faa109e23b0bcb59db6dba623ada9d9156 +Author: Aleksandr Maus +Date: Mon Dec 23 09:59:07 2019 -0500 + + Improve SearchHit "equals" implementation for null fields cases (#50327) (#50448) + + * Improve SearchHit "equals" implementation for null fields cases + +commit cec02da0ac7205ec292f9c9af830827b137c74f1 +Author: Armin Braun +Date: Mon Dec 23 12:24:08 2019 +0100 + + Fix Source Only Snapshot REST Test Failure (#50456) (#50459) + + We are matching on the exact number of shards in this test, but may run into + snapshotting more than the single index created in it due to auto-created indices like + `.watcher`. + Fixed by making the test only take a snapshot of the single index used by this test. + + Closes #50450 + +commit 9a56290ea69c98d2b302f7ff57819f5eb367b188 +Author: Mark Vieira +Date: Fri Dec 20 15:47:23 2019 -0800 + + Update BWC CI versions for 6.8.6 release + +commit dd178d9404bd3209cc6f86d36c10170f6af42086 +Author: Mark Vieira +Date: Fri Dec 20 15:01:19 2019 -0800 + + Upgrade to Gradle Enterprise plugin 3.1.1 (#50451) + +commit 6258d25458fa7db709a6181bda8fcba12c412d60 +Author: Christoph Büscher +Date: Fri Dec 20 22:00:08 2019 +0100 + + Log deprecation for nGram and edgeNGram custom filters (#50376) (#50445) + + The camel-case `nGram` and `edgeNGram` filter names were deprecated in 6. We + currently throw errors on new indices when they are used. However these errors + are currently only thrown for pre-configured filters, adding them as custom + filters doesn't trigger the warning and error. This change adds the appropriate + deprecation warnings for `nGram` and `edgeNGram` respectively on version 7 + indices. + + Relates #50360 + +commit 339d10c16f7bae5cd9994c9e396e5441eba84bba +Author: Igor Motov +Date: Fri Dec 20 04:47:42 2019 -1000 + + Geo: Switch generated GeoJson type names to camel case (#50400) + + Switches generated GeoJson type names to camel case + to conform to the standard. + + Closes #49568 + +commit 1ec89f75dc086d8556b048e7d7aea6326edefa05 +Author: Rory Hunter +Date: Fri Dec 20 20:25:58 2019 +0000 + + Fix NPE when `./gradlew run` without `--data-dir` (#50421) (#50425) + +commit 27ae9a14352d1736272903b5ae8e6e47d277d2cc +Author: James Rodewig +Date: Fri Dec 20 14:02:42 2019 -0500 + + [DOCS] Remove outdated file scripts refererence (#50437) + + File scripts were removed in 6.0 with #24627. + + This removes an outdated file scripts reference from the conditional clauses section of the search templates docs. + +commit c3c9ccf61f8617c9b29e271385cabc74a6868ddf +Author: Lee Hinman +Date: Fri Dec 20 12:33:36 2019 -0700 + + [7.x] Add ILM histore store index (#50287) (#50345) + + * Add ILM histore store index (#50287) + + * Add ILM histore store index + + This commit adds an ILM history store that tracks the lifecycle + execution state as an index progresses through its ILM policy. ILM + history documents store output similar to what the ILM explain API + returns. + + An example document with ALL fields (not all documents will have all + fields) would look like: + + ```json + { + "@timestamp": 1203012389, + "policy": "my-ilm-policy", + "index": "index-2019.1.1-000023", + "index_age":123120, + "success": true, + "state": { + "phase": "warm", + "action": "allocate", + "step": "ERROR", + "failed_step": "update-settings", + "is_auto-retryable_error": true, + "creation_date": 12389012039, + "phase_time": 12908389120, + "action_time": 1283901209, + "step_time": 123904107140, + "phase_definition": "{\"policy\":\"ilm-history-ilm-policy\",\"phase_definition\":{\"min_age\":\"0ms\",\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}},\"version\":1,\"modified_date_in_millis\":1576517253463}", + "step_info": "{... etc step info here as json ...}" + }, + "error_details": "java.lang.RuntimeException: etc\n\tcaused by:etc etc etc full stacktrace" + } + ``` + + These documents go into the `ilm-history-1-00000N` index to provide an + audit trail of the operations ILM has performed. + + This history storage is enabled by default but can be disabled by setting + `index.lifecycle.history_index_enabled` to `false.` + + Resolves #49180 + + * Make ILMHistoryStore.putAsync truly async (#50403) + + This moves the `putAsync` method in `ILMHistoryStore` never to block. + Previously due to the way that the `BulkProcessor` works, it was possible + for `BulkProcessor#add` to block executing a bulk request. This was bad + as we may be adding things to the history store in cluster state update + threads. + + This also moves the index creation to be done prior to the bulk request + execution, rather than being checked every time an operation was added + to the queue. This lessens the chance of the index being created, then + deleted (by some external force), and then recreated via a bulk indexing + request. + + Resolves #50353 + +commit b81e072504b24039a6d64fbf5c97eabd0aebf1a8 +Author: Jack Conradson +Date: Fri Dec 20 10:52:44 2019 -0800 + + Document use of context in put stored script (#50446) + + This documents how to test compile a stored script against a specific + context when using PUT/POST. + +commit 2106a7b02ad6b4b7e237059799afb735e3848727 +Author: Lisa Cawley +Date: Fri Dec 20 10:01:19 2019 -0800 + + [7.x][DOCS] Updates ML links (#50387) (#50409) + +commit a3cdbda7c6a2422b1b99916856fd9549d25b53b7 +Author: Andrei Dan +Date: Fri Dec 20 18:01:03 2019 +0000 + + Make the TransportRolloverAction execute in one cluster state update (#50388) (#50442) + + This commit makes the TransportRolloverAction more resilient, by having it execute + only one cluster state update that creates the new (rollover index), rolls over + the alias from the source to the target index and set the RolloverInfo on the + source index. Before these 3 steps were represented as 3 chained cluster state + updates, which would've seen the user manually intervene if, say, the alias + rollover cluster state update (second in the chain) failed but the creation of + the rollover index (first in the chain) update succeeded + + * Rename innerExecute to applyAliasActions + + (cherry picked from commit 1ba4339a0c73ef3354b8c8b44b628fc55f1dbc78) + Signed-off-by: Andrei Dan + +commit 975cc9951603d2abb2e499e2ca38f14a275f292e +Author: Nhat Nguyen +Date: Fri Dec 20 11:47:47 2019 -0500 + + Close engine before reset log appender (#50390) + + Merge threads can run and access the mock appender after we have stopped it. + + Closes #50315 + +commit 71ff330c4eda5ccbd3906366503f710455065bc0 +Author: Benjamin Trent +Date: Fri Dec 20 12:13:45 2019 -0500 + + [ML][Inference] updates specs with new params + docs (#50373) (#50441) + +commit c37c53a7f77448eac2d10d4065728895edbbd8e0 +Author: Yannick Welsch +Date: Fri Dec 20 17:15:49 2019 +0100 + + Fix testAutoExpandIndicesDuringRollingUpgrade (#50427) + + Fixes the muted test "testAutoExpandIndicesDuringRollingUpgrade". We can't wait in the test for + the index to be green, as we have put a filter exclusion into place that prevents all shards from + being allocated after a node rejoins. Instead we check whether the correct auto-expansion has + taken place. + + Closes #50426 + +commit 9646f3abadfbf469df2e01930f4185ab175c8b85 +Author: Martijn van Groningen +Date: Fri Dec 20 15:43:00 2019 +0100 + + Disable slm in AbstractWatcherIntegrationTestCase (#50422) + + SLM isn't required tests extending from this base class and + only add noise during test suite teardown. + + Closes #50302 + +commit 3e3a93002f1bffb94a60cc471282d651bb165568 +Author: Przemysław Witek +Date: Fri Dec 20 15:34:38 2019 +0100 + + [7.x] Fix accuracy metric (#50310) (#50433) + +commit 14d95aae4642e0b0616c0bfdb89d75f8a0979372 +Author: Przemysław Witek +Date: Fri Dec 20 15:10:33 2019 +0100 + + [7.x] Make each analysis report desired field mappings to be copied (#50219) (#50428) + +commit af8bed13d323ff80e767bf4a6d6d8a75b1a71bfd +Author: Florian Kelbert +Date: Fri Dec 20 13:47:24 2019 +0000 + + [DOCS] Fix typo in bucket sum aggregation docs (#50431) + +commit 5bb668b866feb05938fdacc4a250196510f3bac1 +Author: Przemysław Witek +Date: Fri Dec 20 14:24:23 2019 +0100 + + [7.x] Get rid of maxClassesCardinality internal parameter (#50418) (#50423) + +commit 541dc262bbdbf08f1fa8a300ca4a8c08a8bf42b0 +Author: Alexander Reelsen +Date: Fri Dec 20 13:53:11 2019 +0100 + + Remove accidentally added license files (#50370) + + As license infos and sha files belong to the licenses/ folder, these + files seem to have been added accidentally some time ago. + +commit 2acafd4b1570072502b821f946772cfa6a49c306 +Author: Jim Ferenczi +Date: Fri Dec 20 12:32:37 2019 +0100 + + Optimize composite aggregation based on index sorting (#48399) (#50272) + + Co-authored-by: Daniel Huang + + This is a spinoff of #48130 that generalizes the proposal to allow early termination with the composite aggregation when leading sources match a prefix or the entire index sort specification. + In such case the composite aggregation can use the index sort natural order to early terminate the collection when it reaches a composite key that is greater than the bottom of the queue. + The optimization is also applicable when a query other than match_all is provided. However the optimization is deactivated for sources that match the index sort in the following cases: + * Multi-valued source, in such case early termination is not possible. + * missing_bucket is set to true + +commit 1c7bfebe010dd61bdd37ca8b01baf5eee679fd0b +Author: Yannick Welsch +Date: Fri Dec 20 12:15:43 2019 +0100 + + Mute testAutoExpandIndicesDuringRollingUpgrade + + Relates #50426 + +commit a3837786ec73a7d2333aab199b1fa3eb454c5d9e +Author: Yannick Welsch +Date: Fri Dec 20 11:48:09 2019 +0100 + + Fix testAutoExpandIndicesDuringRollingUpgrade (#50361) + + Follow-up to #50361 that fixes the test that does not work against older ES versions + +commit 4f805deb0ce8cda86799fe5d96dc36d2526deaec +Author: Yannick Welsch +Date: Fri Dec 20 10:22:23 2019 +0100 + + Only auto-expand replicas with allocation filtering when all nodes upgraded (#50361) + + Follow-up to #48974 that ensures that replicas are only auto-expanded according to allocation + filtering rules once all nodes are upgraded to a version that supports this. Helps with + orchestrating cluster upgrades. + +commit 40bce49a7f7960f8a967dc0c4d8ba7a56353c0a6 +Author: Hendrik Muhs +Date: Fri Dec 20 11:25:25 2019 +0100 + + mute SourceDestValidatorTests.testRemoteSourceDoesNotExist + +commit 5f37f1f401874f0526c9625c9efdbe03126691d0 +Author: Yannick Welsch +Date: Fri Dec 20 11:07:18 2019 +0100 + + Revert "Only auto-expand replicas with allocation filtering when all nodes upgraded (#50361)" + + This reverts commit df4fe73b843bf586a302879d89dcf545d8d2f9b8. + +commit 7c10e9b0e71a4042da36fd8011dd05476750b5c5 +Author: Hendrik Muhs +Date: Fri Dec 20 08:28:16 2019 +0100 + + [Transform] improve checkpoint reporting (#50369) + + fixes empty checkpoints, re-factors checkpoint info creation (moves builder) and always reports + last change detection + + relates #43201 + relates #50018 + +commit de14092ad2e698bf1888d1c7a63602d99edc34a8 +Author: Hendrik Muhs +Date: Fri Dec 20 09:55:02 2019 +0100 + + [Transform] refactor source and dest validation to support CCS (#50018) + + refactors source and dest validation, adds support for CCS, makes resolve work like reindex/search, allow aliased dest index with a single write index. + + fixes #49988 + fixes #49851 + relates #43201 + +commit 3cdc23ec9cddd5738c2e9b8211abebf96a05344b +Author: Alan Woodward +Date: Fri Dec 20 09:44:47 2019 +0000 + + Fix meta version of task index mapping (#50363) + + The built-in task index mapping has a version field in its metadata, so that + the TaskResultsService can check to see if it needs to update mappings when + a new task result is stored. #48393 updated this version in TaskResultsService + but omitted to change the version in the mapping itself, so a mapping update + is applied every time a new task result is stored. + + This commit updates the mapping version so that it corresponds to the version + in TaskResultsService. + +commit df4fe73b843bf586a302879d89dcf545d8d2f9b8 +Author: Yannick Welsch +Date: Fri Dec 20 10:22:23 2019 +0100 + + Only auto-expand replicas with allocation filtering when all nodes upgraded (#50361) + + Follow-up to #48974 that ensures that replicas are only auto-expanded according to allocation + filtering rules once all nodes are upgraded to a version that supports this. Helps with + orchestrating cluster upgrades. + +commit f1a6b675f7421796b07e4f6c85249f1e7a83a84a +Author: Marios Trivyzas +Date: Fri Dec 20 00:19:22 2019 +0200 + + SQL: Fix issue with CAST and NULL checking. (#50371) + + Previously, during expression optimisation, CAST would be considered + nullable if the casted expression resulted to a NULL literal, and would + be always non-nullable otherwise. As a result if CASE was wrapped by a + null check function like IS NULL or IS NOT NULL it was simplified to + TRUE/FALSE, eliminating the actual casting operation. So in case of an + expression with an erroneous casting like CAST('foo' AS DATETIME) IS NULL + it would be simplified to FALSE instead of throwing an Exception signifying + the attempt to cast 'foo' to a DATETIME type. + + CAST now always returns Nullability.UKNOWN except from the case that + its result evaluated to a constant NULL, where it returns Nullability.TRUE. + This way the IS NULL/IS NOT NULL don't get simplified to FALSE/TRUE + and the CAST actually gets evaluated resulting to a thrown Exception. + + Fixes: #50191 + (cherry picked from commit 671e07a931cd828661e226cba22a5d38804a17a5) + +commit cb73fb0f9b9db27d0c0d5cc8d876a685631c7ce8 +Author: Tim Brooks +Date: Thu Dec 19 18:02:48 2019 -0700 + + Backport remote proxy mode stats and naming (#50402) + + * Update remote cluster stats to support simple mode (#49961) + + Remote cluster stats API currently only returns useful information if + the strategy in use is the SNIFF mode. This PR modifies the API to + provide relevant information if the user is in the SIMPLE mode. This + information is the configured addresses, max socket connections, and + open socket connections. + + * Send hostname in SNI header in simple remote mode (#50247) + + Currently an intermediate proxy must route conncctions to the + appropriate remote cluster when using simple mode. This commit offers + a additional mechanism for the proxy to route the connections by + including the hostname in the TLS SNI header. + + * Rename the remote connection mode simple to proxy (#50291) + + This commit renames the simple connection mode to the proxy connection + mode for remote cluster connections. In order to do this, the mode specific + settings which we namespaced by their mode (ex: sniff.seed and + proxy.addresses) have been reverted. + + * Modify proxy mode to support a single address (#50391) + + Currently, the remote proxy connection mode uses a list setting for the + proxy address. This commit modifies this so that the setting is + proxy_address and only supports a single remote proxy address. + +commit 878852352da85138796cf584220271d81327d0b2 +Author: Nik Everett +Date: Thu Dec 19 18:55:38 2019 -0500 + + Handle renaming the README (#50404) (#50406) + + We renamed README.textile to README.asciidoc but a bunch of tests and + the package build itself still pointed at the old name. This switches + them the new name. + +commit 2e7686529047cdcf1b03f0e5ab457f6fe0c37d37 +Author: Stuart Tettemer +Date: Thu Dec 19 16:30:34 2019 -0700 + + [DOCS] Deterministic scripted queries are cached (#50408) (#50411) + + **Backport** + + Refs: #49321 + +commit f212994c163445b0b9df274e2564f9dcaa2e1e40 +Author: Stuart Tettemer +Date: Thu Dec 19 16:22:22 2019 -0700 + + [TEST] Unknown scripting annotations raise error (#50343) (#50346) + + Ensure that unknown annotations, such as typo'd `@nondeterministic`, + will raise an exception. + +commit 70ac23ee01c6465b318143e50b9899505f613a3f +Author: Mark Vieira +Date: Thu Dec 19 14:37:25 2019 -0800 + + Disable experimental job trigger + + Signed-off-by: Mark Vieira + +commit 43901b3ef88dff6784ef94298cb9930d8bd02953 +Author: Ryan Ernst +Date: Thu Dec 19 12:56:06 2019 -0800 + + Fix location of README file for rpm/deb + + The file suffix was changed but package building instructions were + missed. + +commit 387139a3fde68d9cba21da66d3012bb003ec6daf +Author: Mark Vieira +Date: Thu Dec 19 12:55:11 2019 -0800 + + Add shell script for performing atomic pushes across branches (#50401) + +commit a0746dbfec06dcbfc7b8cb7e211b5d51824a7b28 +Author: Ryan Ernst +Date: Thu Dec 19 12:47:53 2019 -0800 + + Add --data-dir option to run task (#50342) + + This commit adds a special run.datadir system property that may be + passed to `./gradlew run` which sets the root data directory used by the + task. While normally overriding the data path is not allowed for test + clusters, it is useful when experimenting with the run task. + + closes #50338 + +commit 689df1f28fe0a27e7c7b043b848efd99c7ca0912 +Author: Stuart Tettemer +Date: Thu Dec 19 12:50:25 2019 -0700 + + Scripting: ScriptFactory not required by compile (#50344) (#50392) + + Avoid backwards incompatible changes for 8.x and 7.6 by removing type + restriction on compile and Factory. Factories may optionally implement + ScriptFactory. If so, then they can indicate determinism and thus + cacheability. + + **Backport** + + Relates: #49466 + +commit 6fc7868485415e683c8cef4e0963257d3b321ab3 +Author: James Rodewig +Date: Thu Dec 19 12:58:22 2019 -0500 + + [DOCS] Convert main README to asciidoc (#50303) (#50384) + + Converts the main README file to asciidoc. + + Also includes the following changes: + + * Mentioning Slack instead of IRC + * Removed mentioning of native java API, HTTP should be used + * Removed java as a installed requirement + +commit cc4bc797f9842ba318ceb31d65adb70eae6d36db +Author: Przemysław Witek +Date: Thu Dec 19 18:55:05 2019 +0100 + + [7.x] Implement `precision` and `recall` metrics for classification evaluation (#49671) (#50378) + +commit 903305284dcf98118c0e05c799fb35818e292dbe +Author: Tanguy Leroux +Date: Thu Dec 19 17:50:59 2019 +0100 + + Remove snapshots left by previous tests failures (#50380) + + When a third party test failed, it potentially left some snapshots + in the repository. In case of tests running against an external + service like Azure, the remaining snapshots can fail the future + test executions are they are not supposed to exist. + + Similarly to what has been done for S3 and GCS, this commit + cleans up remaining snapshots before the test execution. + + Closes #50304 + +commit 1a2e931d6ecefd8378f3e381fbef0cbc3313d29c +Author: Alan Woodward +Date: Thu Dec 19 15:11:34 2019 +0000 + + Reduce the max depth of randomly generated interval queries (#50317) + + We randomly generate intervals sources to test serialization and query generation + in IntervalQueryBuilderTests. However, rarely we can generate a query that has + too many nested disjunctions, resulting in query rewrites running afoul of the maximum + boolean clause limit. + + This commit reduces the maximum depth of the randomly generated intervals source + to make running into this limit much more unlikely. + +commit bcae7294566c0272c0543ca4ab48667d155d793e +Author: Benjamin Trent +Date: Thu Dec 19 07:54:20 2019 -0500 + + [ML][Inference] points HLRC javadocs to reference docs (#50321) (#50366) + +commit 1e11d23051927b0577d7b2a2e0652bcb3e5e0432 +Author: Andrei Dan +Date: Thu Dec 19 12:37:07 2019 +0000 + + Extract a create index method that only manipulates the ClusterState (#50240) (#50328) + + * Extract IndexCreationTask execute into applyCreateIndexRequest + + This is the first step in preparation for separating the index creation into a few + steps that only deal with the cluster state mutation and removing the IndexCreationTask + altogether. + + * Split applyCreateIndexRequest + + This breaks down the logic in applyCreateIndexRequest into multiple + steps that will hopefully make the service more readable and unit testable. + The service creation process now goes through a few well defined steps, + namely find the templates that possibly match the new index, parse the + requested and template matching mappings, process the index and template + matching settings, validate the wait for active shards request and + create the `IndexService`, update the mappings in the `MapperService` (which + is grouped together with creating the sort order for validation purposes), + validate the requested and templated matching aliases and finally update + the `ClusterState` to reflect the requested changes. + + This also removes the `IndexCreationTask` as it was a shallow + indirection and migrates the tests from `IndexCreationTaskTests` to + `MetaDataCreateIndexServiceTests` (making them "real" unit tests + operating on the `ClusterState` rather than mocks). + + * Add more unit tests. + + * Add IT to verify we cleanup in case of failure + + (cherry picked from commit 57e6269f750471f05a1a79539ca45361b9e3c2b5) + Signed-off-by: Andrei Dan + + # Conflicts: + # server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java + # server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java + # server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java + +commit 501ab83471df568a335f11af0ff0257323141d90 +Author: István Zoltán Szabó +Date: Thu Dec 19 12:19:44 2019 +0100 + + [DOCS] Adds inference processor documentation (#50204) + + Co-Authored-By: Lisa Cawley + +commit c77ca98928e4441a1124776741989ec26f6cb71d +Author: Igor Motov +Date: Wed Dec 18 07:28:56 2019 -1000 + + Geo: Switch generated WKT to upper case (#50285) + + Switches generated WKT to upper case to + conform to the standard recommendation. + + Relates #49568 + +commit d3c83cd55a20ee57093da8a5660307626b045abe +Author: Dimitris Athanasiou +Date: Wed Dec 18 22:19:59 2019 +0000 + + [7.x][ML] Refresh state index before completing data frame analytics job (#50322) (#50324) + + In order to ensure any persisted model state is searchable by the moment + the job reports itself as `stopped`, we need to refresh the state index + before completing. + + This should fix the occasional failures we see in #50168 and #50313 where + the model state appears missing. + + Closes #50168 + Closes #50313 + + Backport of #50322 + +commit 73ca2e5826e8229f752d44b3462f4826b53e244b +Author: James Rodewig +Date: Wed Dec 18 16:57:38 2019 -0500 + + [DOCS] Document `thread_pool` node stats (#50330) + +commit 9cdbcbd121008702f78a776ac8f4d91c1a9056ac +Author: Stuart Tettemer +Date: Wed Dec 18 14:23:21 2019 -0700 + + [TEST] Exclude name on ScriptContextInfo mutate (#50332) (#50337) + + ScriptContextInfoSerializingTests:testEqualsAndHashcode was failing + because the mutation was generating the same name. + + **Backport** + + Fixes: #50331 + +commit 4396a1f78b8c9cc7ff054182b371cb1b1d224b36 +Author: Benjamin Trent +Date: Wed Dec 18 15:47:06 2019 -0500 + + [ML][Inference] fix support for nested fields (#50258) (#50335) + + This fixes support for nested fields + + We now support fully nested, fully collapsed, or a mix of both on inference docs. + + ES mappings allow the `_source` to be any combination of nested objects + dot delimited fields. + So, we should do our best to find the best path down the Map for the desired field. + +commit 06a24f09cf6973adb16a15ce079de465fabb57ac +Author: Stuart Tettemer +Date: Wed Dec 18 13:00:42 2019 -0700 + + Scripting: Cache script results if deterministic (#50106) (#50329) + + Cache results from queries that use scripts if they use only + deterministic API calls. Nondeterministic API calls are marked in the + whitelist with the `@nondeterministic` annotation. Examples are + `Math.random()` and `new Date()`. + + Refs: #49466 + +commit 246d926412810b70cb259a48f0097867c555dae5 +Author: lcawl +Date: Wed Dec 18 11:46:00 2019 -0800 + + [DOCS] Fixes security links + +commit 35a88a5dbb80c371df7e59bba1af5b1c097ebf3b +Author: Adrien Grand +Date: Wed Dec 18 18:41:26 2019 +0100 + + Add 7.5.2 version. + +commit 8439b2779b26c5f9e7cd9ed30339552aa57d93b3 +Author: Ryan Ernst +Date: Wed Dec 18 09:38:07 2019 -0800 + + Add version 6.8.7 constant + +commit 498532a12d8ab77a1e17de3309ac0b9809a059c2 +Author: James Rodewig +Date: Wed Dec 18 12:17:41 2019 -0500 + + [DOCS] Clarify frozen indices are read-only (#50318) + + The freeze index API docs state that frozen indices are blocked for + write operations. + + While this implies frozen indices are read-only, it does not explicitly + use the term "read-only", which is found in other docs, such as the + force merge docs. + + This adds the "ready-only" term to the freeze index API docs as well + as other clarification. + +commit ef54a9c23c6501186c3f5fe1ed89478db7bc7468 +Author: Nikita Glashenko +Date: Wed Dec 18 20:40:45 2019 +0400 + + Add tests for IntervalsSourceProvider.Wildcard and Prefix (#50306) + + This PR adds unit tests for wire and xContent serialization of `IntervalsSourceProvider.Wildcard` + and `IntervalsSourceProvider.Prefix`. + + Relates #50150 + +commit bf63f24209732b737e9ea3b054865fbc9c10a063 +Author: Christoph Büscher +Date: Wed Dec 18 17:35:35 2019 +0100 + + [Docs] Remove `intervals` filter rule from allowed top-level rules (#50320) + + The `filter` rule is not allowed on the top-level of the query, so removing it + from the list of allowed rules. Where it can be nested inside other rules, those + rules already mention it. + +commit ec11c04bd5b3fa29e7a315d84bfa5c96e74ed00b +Author: Adrien Grand +Date: Wed Dec 18 17:08:37 2019 +0100 + + [DOCS] Add 7.5.1 release notes (#50196) + + Co-Authored-By: Lisa Cawley + +commit cbffd127d533aa0d7889a400d14005d57229e3fd +Author: Kevin Woblick +Date: Wed Dec 18 15:03:45 2019 +0100 + + [DOCS] Add warning about Docker port exposure (#50169) + + Docker bypasses the Uncomplicated Firewall (UFW) on Linux by editing the `iptables` config directly, which leads to the exposure of port 9200, even if you blocked it via UFW. + + This adds a warning along with work-arounds to the docs. + + Signed-off-by: Kovah + +commit 7c5a3bcf6ddf91553644cc2be992b0614cec17b6 +Author: Jason Tedor +Date: Wed Dec 18 08:26:57 2019 -0500 + + Always consume the body in has privileges (#50298) + + Our REST infrastructure will reject requests that have a body where the + body of the request is never consumed. This ensures that we reject + requests on endpoints that do not support having a body. This requires + cooperation from the REST handlers though, to actually consume the body, + otherwise the REST infrastructure will proceed with rejecting the + request. This commit addresses an issue in the has privileges API where + we would prematurely try to reject a request for not having a username, + before consuming the body. Since the body was not consumed, the REST + infrastructure would instead reject the request as a bad request. + +commit 447bac27d2ae75997fb82cca92a8813d5375ab59 +Author: Dimitris Athanasiou +Date: Wed Dec 18 12:30:11 2019 +0000 + + [7.x][ML] Delete unused data frame analytics state (#50243) (#50280) + + This commit adds removal of unused data frame analytics state + from the _delete_expired_data API (and in extend th ML daily + maintenance task). At the moment the potential state docs + include the progress document and state for regression and + classification analyses. + + Backport of #50243 + +commit 82086929d78fe67576e0c878074f7664fc96bb64 +Author: Yannick Welsch +Date: Wed Dec 18 10:36:41 2019 +0100 + + Increase timeout on FollowIndexSecurityIT.testAutoFollowPatterns (#50282) + + This test was causing test failures on slow CI runs. + + Closes #50279 + +commit 5759a263cbcebbe5bdb4b2747448536b17c0ec96 +Author: István Zoltán Szabó +Date: Wed Dec 18 09:10:12 2019 +0100 + + [DOCS] Adds GET, GET stats and DELETE inference APIs (#50224) + + Co-Authored-By: Lisa Cawley + +commit ac974c35c00282750dcd2f3dfb3bec652d80ee9f +Author: Przemysław Witek +Date: Tue Dec 17 21:32:37 2019 +0100 + + Pass processConnectTimeout to the method that fetches C++ process' PID (#50276) (#50290) + +commit 30d66828aebe5417ffdd6c961540be629d14c3d9 +Author: Lisa Cawley +Date: Tue Dec 17 09:01:31 2019 -0800 + + [DOCS] Move transform resource definitions into APIs (#50108) + +commit e3459637a7d5ad8daabe7bc63da09058accae032 +Author: Ryan Ernst +Date: Tue Dec 17 09:52:16 2019 -0800 + + Ensure global buildinfo plugin is applied for distro download (#50249) + + This commit ensures the global info plugin is applied, which supplies + the isInternal flag used to determine whether distro download looks for + bwcVersions. + + relates #50230 + +commit afe9ee3fa5bdadd0694918b0277efdd087b27f28 +Author: Florian Kelbert +Date: Tue Dec 17 16:08:38 2019 +0000 + + [DOCS] Fix typo in Create API key docs (#50233) + +commit 726c35dfd04a30a996802f977aeb353b5bcc754e +Author: James Rodewig +Date: Tue Dec 17 09:31:07 2019 -0500 + + [DOCS] Add identifier mapping tip to numeric and keyword datatype docs (#49933) + + Users often mistakenly map numeric IDs to numeric datatypes. However, + this is often slow for the `term` and other term-level queries. + + The "Tune for search speed" docs includes advice for mapping numeric + IDs to `keyword` fields. However, this tip is not included in the + `numeric` or `keyword` field datatype doc pages. + + This rewords the tip in the "Tune for search speed" docs, relocates it + to the `numeric` field docs, and reuses it using tagged regions. + +commit 55cc5432d690338a8fdc2d6073eeca0804a91090 +Author: Armin Braun +Date: Tue Dec 17 15:33:09 2019 +0100 + + Fix S3 Repo Tests Incomplete Reads (#50268) (#50275) + + We need to read in a loop here. A single read to a huge byte array will + only read 16k max with the S3 SDK so if the blob we're trying to fully + read is larger we close early and fail the size comparison. + Also, drain streams fully when checking existence to avoid S3 SDK warnings. + +commit 99fdea50dd71aef45af1811cece8993da8cad8d8 +Author: James Rodewig +Date: Tue Dec 17 09:23:38 2019 -0500 + + [DOCS] Correct cat snapshots API request example (#50274) + + For 7.x and earlier branches, `_cat/repositories` API requests require a + repository name. + + This removes an erroneous request example without a repository name + added with a8e0275. + +commit 37b8c139b3de87b314643ecdc137e6f9f9c4a20f +Author: Yannick Welsch +Date: Tue Dec 17 13:52:39 2019 +0100 + + Omit loading IndexMetaData when inspecting shards (#50214) + + Loading shard state information during shard allocation sometimes runs into a situation where a + data node does not know yet how to look up the shard on disk if custom data paths are used. + The current implementation loads the index metadata from disk to determine what the custom + data path looks like. This PR removes this dependency, simplifying the lookup. + + Relates #48701 + +commit 1f981580aa58e111937b6a28b9efdc3bd449765e +Author: Yannick Welsch +Date: Tue Dec 17 13:51:51 2019 +0100 + + Simplify InternalTestCluster.fullRestart (#50218) + + With node ordinals gone, there's no longer a need for such a complicated full cluster restart + procedure (as we can now uniquely associate nodes to data folders). + + Follow-up to #41652 + +commit 098f540f9d393f673dcc3e415fb21dc2afb789ff +Author: David Kyle +Date: Tue Dec 17 13:03:27 2019 +0000 + + [ML] Remove usage of base action logger in ml actions (#50074) (#50236) + +commit 2079f1cbeb26f2d93a028c4c351cf6a07648a888 +Author: Martijn van Groningen +Date: Tue Dec 17 12:27:07 2019 +0100 + + Backport: Fix ingest simulate response document order if processor executes async (#50269) + + Backport #50244 to 7.x branch. + + If a processor executes asynchronously and the ingest simulate api simulates with + multiple documents then the order of the documents in the response may not match + the order of the documents in the request. + + Alexander Reelsen discovered this issue with the enrich processor with the following reproduction: + + ``` + PUT cities/_doc/munich + {"zip":"80331","city":"Munich"} + + PUT cities/_doc/berlin + {"zip":"10965","city":"Berlin"} + + PUT /_enrich/policy/zip-policy + { + "match": { + "indices": "cities", + "match_field": "zip", + "enrich_fields": [ "city" ] + } + } + + POST /_enrich/policy/zip-policy/_execute + + GET _cat/indices/.enrich-* + + POST /_ingest/pipeline/_simulate + { + "pipeline": { + "processors" : [ + { + "enrich" : { + "policy_name": "zip-policy", + "field" : "zip", + "target_field": "city", + "max_matches": "1" + } + } + ] + }, + "docs": [ + { "_id": "first", "_source" : { "zip" : "80331" } } , + { "_id": "second", "_source" : { "zip" : "50667" } } + ] + } + ``` + + * fixed test compile error + +commit 4f24739fbefcb996ff612931b80c04d7807ae046 +Author: Armin Braun +Date: Tue Dec 17 10:58:15 2019 +0100 + + Fix Index Deletion During Partial Snapshot Create (#50234) (#50266) + + We can simply filter out shard generation updates for indices + that were removed from the cluster state concurrently to fix + index deletes during partial snapshots as that completely removes + any reference to those shards from the snapshot. + + Follow up to #50202 + Closes #50200 + +commit 2e7b1ab375a058ff478524f6db15d7742161df29 +Author: Armin Braun +Date: Tue Dec 17 10:55:13 2019 +0100 + + Use ClusterState as Consistency Source for Snapshot Repositories (#49060) (#50267) + + Follow up to #49729 + + This change removes falling back to listing out the repository contents to find the latest `index-N` in write-mounted blob store repositories. + This saves 2-3 list operations on each snapshot create and delete operation. Also it makes all the snapshot status APIs cheaper (and faster) by saving one list operation there as well in many cases. + This removes the resiliency to concurrent modifications of the repository as a result and puts a repository in a `corrupted` state in case loading `RepositoryData` failed from the assumed generation. + +commit ce294e1564d1525599e6ce493ef760823dd95268 +Author: Armin Braun +Date: Tue Dec 17 09:42:39 2019 +0100 + + Better Logging S3 Bulk Delete Failures (#50203) (#50262) + + Unfortunately bulk delete exceptions don't show the individual delete + errors when a bulk delete fails when you log them outright so I added this work-around + to get the individual details to get useful logging. + +commit c6fdf9ed8aee9b210eed2dae632f422861ff87f7 +Author: Andrei Stefan +Date: Tue Dec 17 10:00:54 2019 +0200 + + Handle NULL in ResultSet's getDate() method (#50184) + + (cherry picked from commit 08214eb1338fef5c8082c3f8b84c24dd53224ebe) + +commit ce2aab3f2f32323a11dcc95b2a3e4741d1d38515 +Author: Tim Vernum +Date: Tue Dec 17 14:58:58 2019 +1100 + + Add setting to restrict license types (#50252) + + This adds a new "xpack.license.upload.types" setting that restricts + which license types may be uploaded to a cluster. + + By default all types are allowed (excluding basic, which can only be + generated and never uploaded). + This setting does not restrict APIs that generate licenses such as the + start trial API. + + This setting is not documented as it is intended to be set by + orchestrators and not end users. + + Backport of: #49418 + +commit 0efb241b3c13cacbb97da01638ea408ee409fd07 +Author: Przemko Robakowski +Date: Tue Dec 17 01:15:15 2019 +0100 + + Fix flakiness in CsvProcessorTests (#50254) (#50256) + + There's flakiness in CsvProcesorTests, where tests fail if random document generator add field that should not be present. This change cleans generated document from these problematic fields. + + Closes #50209 + +commit 8391b974c53ecedb1e59fc593aacd16e96830a8f +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Dec 16 21:59:54 2019 +0100 + + Recovery buffer size 16B smaller (#50100) + + G1GC will use humongous allocations when an allocation exceeds half the + chosen region size, which is minimum 1MB. By reducing the recovery + buffer size by 16 bytes we ensure that the recovery buffer is never + allocated as a humongous allocation. + +commit 620b84c84de0e19756b7fe593f9f9d8584d511dd +Author: Jason Tedor +Date: Mon Dec 16 15:07:38 2019 -0500 + + Respect ES_PATH_CONF during upgrades too (#50246) + + A previous commit taught Elasticsearch packages to respect ES_PATH_CONF + during installs. Missed in that commit was respecting ES_PATH_CONF on + upgrades. This commit does that. Additionally, while ES_PATH_CONF is not + currently used in pre-install, this commit adds respect to the preinst + script in case we do in the future. + +commit 463cd414aaaf03aed6e2ccb74ed09a9af8ff56c8 +Author: Julie Tibshirani +Date: Mon Dec 16 10:37:03 2019 -0800 + + Bump the scroll keep-alive time in cluster upgrade tests. (#50195) + + In the yaml cluster upgrade tests, we start a scroll in a mixed-version cluster, + then attempt to continue the scroll after the upgrade is complete. This test + occasionally fails because the scroll can expire before the cluster is done + upgrading. + + The current scroll keep-alive time 5m. This PR bumps it to 10m, which gives a + good buffer since in failing tests the time was only exceeded by ~30 seconds. + + Addresses #46529. + +commit 731bfa6614be12fc2a1ccbd43b91e49262e67abf +Author: Nhat Nguyen +Date: Mon Dec 16 11:08:00 2019 -0500 + + Account trimAboveSeqNo in committed translog generation (#50205) + + Today we do not consider trimAboveSeqNo when calculating the translog + generation of an index commit. If there is no new indexing after the + primary promotion, then we won't be able to clean up the translog. + +commit be78d5cc74cc17938f948a6b06e361de8f566e6c +Author: Zachary Tong +Date: Mon Dec 16 11:14:45 2019 -0500 + + Migrate MinAggregator integration tests to AggregatorTestCase (#50053) + + Also renames MinTests to MinAggregationBuilderTests + +commit 2bd3a0589259ffda6caf27bf23bb8ed8813bc4e9 +Author: Rory Hunter +Date: Mon Dec 16 15:39:28 2019 +0000 + + Refactor environment variable processing for Docker (#50221) + + Backport of #49612. + + The current Docker entrypoint script picks up environment variables and + translates them into -E command line arguments. However, since any tool + executes via `docker exec` doesn't run the entrypoint, it results in + a poorer user experience. + + Therefore, refactor the env var handling so that the -E options are + generated in `elasticsearch-env`. These have to be appended to any + existing command arguments, since some CLI tools have subcommands and + -E arguments must come after the subcommand. + + Also extract the support for `_FILE` env vars into a separate script, so + that it can be called from more than once place (the behaviour is + idempotent). + + Finally, add noop -E handling to CronEvalTool for parity, and support + `-E` in MultiCommand before subcommands. + +commit 55426862834a60ce9189dff662131a243cfd9957 +Author: David Kyle +Date: Mon Dec 16 14:55:01 2019 +0000 + + [ML] Wait for green after opening job in NetworkDisruptionIT (#50232) + + Closes #49908 + +commit 32b24457443b0f2c70947f08048ec8b093c7c357 +Author: David Roberts +Date: Mon Dec 16 14:12:36 2019 +0000 + + Change process kill order for testclusters shutdown (#50215) + + The testclusters shutdown code was killing child processes + of the ES JVM before the ES JVM. This causes any running + ML jobs to be recorded as failed, as the ES JVM notices that + they have disconnected from it without being told to stop, + as they would if they crashed. In many test suites this + doesn't matter because the test cluster will never be + restarted, but in the case of upgrade tests it makes it + impossible to test what happens when an ML job is running + at the time of the upgrade. + + This change reverses the order of killing the ES process + tree such that the parent processes are killed before their + children. A list of children is stored before killing the + parent so that they can subsequently be killed (if they + don't exit by themselves as a side effect of the parent + dying). + + Backport of #50175 + +commit d934f8ff100ea58851bcf1a0982856c969f9dce0 +Author: Andrew Odendaal +Date: Mon Dec 16 13:30:05 2019 +0000 + + [Docs] Fix typo in README (#50229) + +commit afcdc27c0242260cbb2fed890ba836ff56679bd0 +Author: Armin Braun +Date: Mon Dec 16 13:30:05 2019 +0100 + + Fix Index Deletion during Snapshot Finalization (#50202) (#50227) + + With #45689 making it so that index metadata is written + after all shards have been snapshotted we can't delete indices + that are part of the upcoming snapshot finalization any longer + and it is not sufficient to check if all shards of an index have been + snapshotted before deciding that it is safe to delete it. + This change forbids deleting any index that is in the process of being + snapshot to avoid issues during snapshot finalization. + + Relates #50200 (doesn't fully fix yet because we're not fixing the `partial=true` + snapshot case here + +commit 4ced237a7f9e9f4ea154414ad42acb7cebb35aa9 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Dec 16 12:42:42 2019 +0100 + + Disk threshold decider is enabled by default (#50222) + + An old comment had survived after the default was flipped. + + Relates #6204 + +commit 73add726d758f6b7d1d23e0e9cbc8c7da67f0045 +Author: Dimitris Athanasiou +Date: Mon Dec 16 11:30:06 2019 +0000 + + [7.x][ML] Fix exception when field is not included and excluded at the same time (#50192) (#50223) + + Executing the data frame analytics _explain API with a config that contains + a field that is not in the includes list but at the same time is the excludes + list results to trying to remove the field twice from the iterator. That causes + an `IllegalStateException`. This commit fixes this issue and adds a test that + captures the scenario. + + Backport of #50192 + +commit 761d6e8e4b21529427667d0e5bc6f2f8c083fa29 +Author: Armin Braun +Date: Mon Dec 16 11:37:09 2019 +0100 + + Remove BlobContainer Tests against Mocks (#50194) (#50220) + + * Remove BlobContainer Tests against Mocks + + Removing all these weird mocks as asked for by #30424. + All these tests are now part of real repository ITs and otherwise left unchanged if they had + independent tests that didn't call the `createBlobStore` method previously. + The HDFS tests also get added coverage as a side-effect because they did not have an implementation + of the abstract repository ITs. + + Closes #30424 + +commit 3717c733ff18bd29e18a8f83bb96a1fab32cb9d2 +Author: Ignacio Vera +Date: Mon Dec 16 09:17:51 2019 +0100 + + "CONTAINS" support for BKD-backed geo_shape and shape fields (#50141) (#50213) + + Lucene 8.4 added support for "CONTAINS", therefore in this commit those + changes are integrated in Elasticsearch. This commit contains as well a + bug fix when querying with a geometry collection with "DISJOINT" relation. + +commit c732d9923d45145b582588e0b0e4f7d7235322e4 +Author: Nhat Nguyen +Date: Sun Dec 15 21:37:50 2019 -0500 + + Fix doc type in FullClusterRestartIT + + "_doc" is not accepted in 6.x version. + +commit a9d16ee895befc3209df93d87f928fbe1703d266 +Author: Tim Vernum +Date: Mon Dec 16 10:11:21 2019 +1100 + + Skip enterprise license tests in release build (#50182) + + The release builds use a production license key, and our rest test load + licenses that are signed by the dev license key. + + This change adds the new enterprise license Rest tests to the + blacklist on release builds. + + Backport of: #50163 + +commit 4d22e3cd156482be283ed2c509d9a78db33e4743 +Author: Nhat Nguyen +Date: Sun Dec 15 13:18:13 2019 -0500 + + Skip bwc versions without retention leases + +commit 6f1098cceb18ea72ccebba2fc8d07049eb58556a +Author: Nhat Nguyen +Date: Sun Dec 15 12:57:54 2019 -0500 + + Fix version in testTurnOffTranslogRetentionAfterAllShardStarted + + Soft-deletes requires 6.5 or later. + +commit a0044df78a92242c6c2887f214383901e7d32643 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Sun Dec 15 17:30:28 2019 +0100 + + Reindex source types disregarded in 7.x (#49580) + + Clarify that types in source index are disregarded. + +commit df46848fb09b1da3ea350c95f2843f6d1e1ac314 +Author: Nhat Nguyen +Date: Fri Dec 13 13:56:50 2019 -0500 + + Migrate peer recovery from translog to retention lease (#49448) + + Since 7.4, we switch from translog to Lucene as the source of history + for peer recoveries. However, we reduce the likelihood of + operation-based recoveries when performing a full cluster restart from + pre-7.4 because existing copies do not have PPRL. + + To remedy this issue, we fallback using translog in peer recoveries if + the recovering replica does not have a peer recovery retention lease, + and the replication group hasn't fully migrated to PRRL. + + Relates #45136 + +commit c151a75dfe065099289aa14e26cf3e7507d6c4b4 +Author: Nhat Nguyen +Date: Thu Nov 21 14:57:38 2019 -0500 + + Use retention lease in peer recovery of closed indices (#48430) + + Today we do not use retention leases in peer recovery for closed indices + because we can't sync retention leases on closed indices. This change + allows that ability and adjusts peer recovery to use retention leases + for all indices with soft-deletes enabled. + + Relates #45136 + + Co-authored-by: David Turner + +commit bbb872a022d59525a8ede957b2e2c80207d5e754 +Author: James Rodewig +Date: Fri Dec 13 12:58:11 2019 -0500 + + Revert "[DOCS] Add `index-extra-title-page.html` for direct HTML migration (#50189)" + + This reverts commit 79c40728f7c7f760eaa4b031ef80171ced350f3c. + +commit 79c40728f7c7f760eaa4b031ef80171ced350f3c +Author: James Rodewig +Date: Fri Dec 13 12:44:12 2019 -0500 + + [DOCS] Add `index-extra-title-page.html` for direct HTML migration (#50189) + +commit 4805d8ac7df5a65b3f66740a3ce6f59541d6a6ef +Author: Benjamin Trent +Date: Fri Dec 13 10:39:51 2019 -0500 + + [ML][Inference] Adding a warning_field for warning msgs. (#49838) (#50183) + + This adds a new field for the inference processor. + + `warning_field` is a place for us to write warnings provided from the inference call. When there are warnings we are not going to write an inference result. The goal of this is to indicate that the data provided was too poor or too different for the model to make an accurate prediction. + + The user could optionally include the `warning_field`. When it is not provided, it is assumed no warnings were desired to be written. + + The first of these warnings is when ALL of the input fields are missing. If none of the trained fields are present, we don't bother inferencing against the model and instead provide a warning stating that the fields were missing. + + Also, this adds checks to not allow duplicated fields during processor creation. + +commit 41736dd6c35ad13fcc51aa8df835fb39e90186c2 +Author: Benjamin Trent +Date: Fri Dec 13 10:39:34 2019 -0500 + + [ML] retry bulk indexing of state docs (#50149) (#50185) + + This exchanges the direct use of the `Client` for `ResultsPersisterService`. State doc persistence will now retry. Failures to persist state will still not throw, but will be audited and logged. + +commit fe3c9e71d12e9f7c79856438f897280ee65e853f +Author: Dimitris Athanasiou +Date: Fri Dec 13 17:00:55 2019 +0200 + + [7.x][ML] Fix DFA explain API timeout when source index is missing (#50176) (#50180) + + This commit fixes a bug that caused the data frame analytics + _explain API to time out in a multi-node setup when the source + index was missing. When we try to create the extracted fields detector, + we check the index settings. If the index is missing that responds + with a failure that could be wrapped as a remote exception. + While we unwrapped correctly to check if the cause was an + `IndexNotFoundException`, we then proceeded to cast the original + exception instead of the cause. + + Backport of #50176 + +commit cd04021961515309e1cc47018f147899cabd80a4 +Author: James Rodewig +Date: Fri Dec 13 08:43:35 2019 -0500 + + [DOCS] Reformat token count limit filter docs (#49835) + +commit 46376100b14681681a5f8cbe4fedb3b32aecc5c0 +Author: Ioannis Kakavas +Date: Fri Dec 13 13:46:44 2019 +0200 + + Fix testMalformedToken (#50164) (#50170) + + This test was fixed as part of #49736 so that it used a + TokenService mock instance that was enabled, so that token + verification fails because the token is invalid and not because + the token service is not enabled. + When the randomly generated token we send, decodes to being of + version > 7.2 , we need to have mocked a GetResponse for the call + that TokenService#getUserTokenFromId will make, otherwise this + hangs and times out. + +commit 8f36bfa37f301eafb26c97f184aa4f22ff6ef2ba +Author: István Zoltán Szabó +Date: Fri Dec 13 12:22:50 2019 +0100 + + [7.x][DOCS] Changes hyperparam optimization section ID (#50173) + +commit 7611b3c9be63aedb06acd859959f29c88bc8b300 +Author: István Zoltán Szabó +Date: Fri Dec 13 11:48:21 2019 +0100 + + [7.x][DOCS] Moves data frame analytics job resource definitions into APIs (#50165) + + * [7.x][DOCS] Moves data frame analytics job resource definitions into APIs. + +commit ac83e45a6b75fa0e9daa96647e5cc6e4dee1d624 +Author: Ioannis Kakavas +Date: Fri Dec 13 12:47:53 2019 +0200 + + Disable TLS diagnostics in FIPS mode (#49900) + + This commit sets xpack.security.ssl.diagnose.trust to false in all + the nodes of our TestClusters when running integTest. This is needed + in 7.x because setting xpack.security.ssl.diagnose.trust to true + wraps SunJSSE TrustManager with our own DiagnosticTrustManager and + this is not allowed when SunJSSE is in FIPS mode. + An alternative would be to set `xpack.security.fips.enabled` to + true which would also implicitly disable + xpack.security.ssl.diagnose.trust but would have additional effects + (would require that we set PBKDF2 for password hashing algorithm in + all test clusters, would prohibit using JKS keystores in nodes even + if relevant tests have been muted in FIPS mode etc.) + +commit c0216f9a06eee4ec9e729c3c3d1b70d6a2eca2b4 +Author: Christoph Büscher +Date: Fri Dec 13 09:59:45 2019 +0100 + + Improve DateFieldMapper `ignore_malformed` handling (#50090) + + A recent change around date parsing (#46675) made it stricter, so we should now also + catch DateTimeExceptions in DateFieldMapper and ignore those when the + `ignore_malformed` option is set. + + Closes #50081 + +commit e6cbcf7f7c9118955eafb269f5feb774eb41994b +Author: Dimitris Athanasiou +Date: Fri Dec 13 10:33:19 2019 +0200 + + [7.x] [ML] Persist/restore state for DFA classification (#50040) (#50147) + + This commit adds state persist/restore for data frame analytics classification jobs. + + Backport of #50040 + +commit 1c3ce110bdcdd164864dd78429342814ffe5bd8c +Author: Hendrik Muhs +Date: Fri Dec 13 08:09:25 2019 +0100 + + [Transform] add actual timeout in message (#50140) + + add the timeout to the message if stopping a transform times out + +commit efa6ef3c06eabb331821837d7919f19b18459886 +Author: Jason Tedor +Date: Thu Dec 12 22:26:35 2019 -0500 + + Respect ES_PATH_CONF on package install (#50158) + + We respect ES_PATH_CONF everywhere except package install. This commit + addresses this by respecting ES_PATH_CONF when installing the RPM/Debian + packages. + +commit 29526d0dfe8e9333121451b71c31a43e0e7ace7f +Author: Jason Tedor +Date: Thu Dec 12 19:17:59 2019 -0500 + + Validate exporter type is HTTP for HTTP exporter (#49992) + + Today the HTTP exporter settings without the exporter type having been + configured to HTTP. When it is time to initialize the exporter, we can + blow up. Since this initialization happens on the cluster state applier + thread, it is quite problematic that we do not reject settings updates + where the type is not configured to HTTP, but there are HTTP exporter + settings configured. This commit addresses this by validating that the + exporter type is not only set, but is set to HTTP. + +commit 5b9fce48c0f3f9516eedaf01a788e96a1771b1ee +Author: Mark Vieira +Date: Thu Dec 12 15:21:28 2019 -0800 + + Trim minimumGradleVersion whitespace before parsing + +commit 2811b97b7636f4438d56443c3584f328f2e7ab26 +Author: Tim Vernum +Date: Fri Dec 13 10:22:55 2019 +1100 + + Remove reserved roles for code search (#50115) + + The "code_user" and "code_admin" reserved roles existed to support + code search which is no longer included in Kibana. + + The "kibana_system" role included privileges to read/write from the + code search indices, but no longer needs that access. + + Backport of: #50068 + +commit e287427a039246e8dcc79a83314251ed3eb0c76f +Author: Mark Vieira +Date: Thu Dec 12 14:52:36 2019 -0800 + + Don't pass OPENSHIFT_IP env variable when building old BWC branches (#50153) + + This commit tweaks the workaround introduced in #49211 to support + Gradle 6.0. In the workaround, we specifically override the address + the Gradle daemon binds to by passing the desired address via the + OPENSHIFT_IP environment variable. This works fine for builds using + Gradle 6.0, but for older Gradle versions this causes issues with + inter-daemon communication, specifically when we build BWC branches + not on Gradle 6.0. The fix here is to strip that environment variable + out when building the target BWC branch if that branch is on an + older Gradle version. + + This is all temporary and will be removed when this bug fix is released + in Gradle 6.1. + + Closes #50025 + +commit 249334f38d0babf52583c3f854bdb0a35b0a3975 +Author: James Rodewig +Date: Thu Dec 12 15:41:20 2019 -0500 + + [DOCS] Document JVM node stats (#49500) + + * [DOCS] Document JVM node stats + + Documents the `jvm` parameters returned by the `_nodes/stats` API. + + Co-Authored-By: James Baiera + +commit 73c412063b74178bb895a9d01fd6996be011511b +Author: Julie Tibshirani +Date: Thu Dec 12 11:34:24 2019 -0800 + + Reenable the 'continue scroll' cluster upgrade test. + +commit 2d19d0c16691bfd8bef92ec1849671f70aba8fcc +Author: Lisa Cawley +Date: Thu Dec 12 08:20:39 2019 -0800 + + [DOCS] Updates transform screenshots and text (#50059) + +commit d7ffa7f8f72140afab49874ded9c823f3a9d9f58 +Author: Benjamin Trent +Date: Thu Dec 12 12:24:58 2019 -0500 + + [7.x][ML] Add graceful retry for anomaly detector result indexing failures(#49508) (#50145) + + * [ML] Add graceful retry for anomaly detector result indexing failures (#49508) + + All results indexing now retry the amount of times configured in `xpack.ml.persist_results_max_retries`. The retries are done in a semi-random, exponential backoff. + + * fixing test + +commit f42537a0f4eacb5911633cfc4b6f2fc18c1a2a32 +Author: Mark Vieira +Date: Wed Dec 11 13:01:54 2019 -0800 + + Fall back to Java 13 APIs for forbidden API checks when using JDK 14 (#50095) + + Closes #50041 + + (cherry picked from commit 4745b77ddefcaa5b00e700bd8e3ec7752f03c52d) + +commit c043aa887fbb560336a9817bb02f4ed90cff3d00 +Author: Benjamin Trent +Date: Thu Dec 12 11:13:55 2019 -0500 + + [ML][Inference] Simplify inference processor options (#50105) (#50146) + + * [ML][Inference] Simplify inference processor options + + * addressing pr comments + +commit 13e47df97df66acf0259ca607aeaa728d5783f1e +Author: David Roberts +Date: Thu Dec 12 15:33:22 2019 +0000 + + [TEST] Increase timeout for ML internal cluster cleanup (#50142) + + Closes #48511 + +commit 1186a5dc09432caa8041e2b680136997188a53c8 +Author: James Rodewig +Date: Thu Dec 12 09:39:06 2019 -0500 + + [DOCS] Reformat lowercase token filter docs (#49935) + +commit 521933aa111cd22a0dec89cdfa40cebde79f6ace +Author: Zachary Tong +Date: Thu Dec 12 09:06:33 2019 -0500 + + SingleBucket aggs need to reduce their bucket's pipelines first (#50103) + + When decoupling the pipeline reduction from regular agg reduction, + MultiBucket aggs were modified to reduce their bucket's pipeline + aggs first before reducing the sibling aggs. This modification + was missed on SingleBucket aggs, meaning any SingleBucket would + fail to reduce any pipeline sub-aggs + +commit 364eb2d34cf435b1d405768a8b9b58b37c48b6ba +Author: James Rodewig +Date: Thu Dec 12 08:38:48 2019 -0500 + + [DOCS] Correct percentile rank agg example response (#50052) + + The example snippets in the percentile rank agg docs use a test dataset + named `latency`, which is generated from docs/gradle.build. + + At some point the dataset and example snippets were updated, but the + text surrounding the snippets was not. This means the text and the + example snippets shown no longer match up. + + This corrects that by changing the snippets using /TESTRESPONSE magic comments. + +commit c73930988bda1860efecb0e40a0c265c5f91a850 +Author: Armin Braun +Date: Thu Dec 12 14:18:06 2019 +0100 + + Remove Unused Delete Endpoint from GCS Mock (#50128) (#50134) + + Follow up to #50024: we're not using the single-delete any more so no need to have a mock endpoint for it + +commit c2adc4d7563687bc9c2b16d71bb84477883b5792 +Author: Martijn Laarman +Date: Thu Dec 12 13:12:04 2019 +0100 + + cat.indices.json bytes enum not exhaustive (#49369) + + Missing several valid options that other cat API's do define + + (cherry picked from commit 7bb4781bc9966bcb6f9275920b2158fb959a30aa) + +commit b5ec227de8eb711a63aad7ea272e596128890b18 +Author: Ignacio Vera +Date: Thu Dec 12 13:13:37 2019 +0100 + + upgrade to lucene 8.4.0-snapshot-08b8d116f8f (#50129) (#50132) + +commit 7d4118dc4e847315f9693eef73bda63f89f78420 +Author: David Kyle +Date: Thu Dec 12 11:03:59 2019 +0000 + + Enable trace logging in failing ml NetworkDisruptionIT + + https://github.com/elastic/elasticsearch/issues/49908 + +commit 0bba7ccedd5c255e2f8eb9c6aafe37d26bbb27d6 +Author: Adrien Grand +Date: Thu Dec 12 11:46:37 2019 +0100 + + Remove information about the latest PostingsFormat/DocValuesFormat. (#50118) (#50127) + + This information is outdated and unused. + +commit 2a186c8148f18a005cd30f3eb8a9cba015443e52 +Author: Armin Braun +Date: Thu Dec 12 11:40:44 2019 +0100 + + Disable LongGCDisruptionTests on JDK11+12 (#50097) (#50126) + + See discussion in #50047 (comment). + There are reproducible issues with Thread#suspend in Jdk11 and Jdk12 for me locally + and we have one failure for each on CI. + Jdk8 and Jdk13 are stable though on CI and in my testing so I'd selectively disable + this test here to keep the coverage. We aren't using suspend in production code so the + JDK bug behind this does not affect us. + + Closes #50047 + +commit 6eee41e253605d993c3500e5fae229c7ad70a0ca +Author: Armin Braun +Date: Thu Dec 12 11:17:46 2019 +0100 + + Remove Unused Single Delete in BlobStoreRepository (#50024) (#50123) + + * Remove Unused Single Delete in BlobStoreRepository + + There are no more production uses of the non-bulk delete or the delete that throws + on missing so this commit removes both these methods. + Only the bulk delete logic remains. Where the bulk delete was derived from single deletes, + the single delete code was inlined into the bulk delete method. + Where single delete was used in tests it was replaced by bulk deleting. + +commit 0fae4065effbcd83e172f5ba56c0a1f2eba7e824 +Author: Armin Braun +Date: Thu Dec 12 11:17:22 2019 +0100 + + Better Logging GCS Blobstore Mock (#50102) (#50124) + + * Better Logging GCS Blobstore Mock + + Two things: + 1. We should just throw a descriptive assertion error and figure out why we're not reading a multi-part instead of + returning a `400` and failing the tests that way here since we can't reproduce these 400s locally. + 2. We were missing logging the exception on a cleanup delete failure that coincides with the `400` issue in tests. + + Relates #49429 + +commit cbff63685acd4dfc7a16117d1deed0126c933a16 +Author: Ryan Ernst +Date: Wed Dec 11 21:30:56 2019 -0800 + + Ensure meta and document field maps are never null in GetResult (#50112) + + This commit ensures deseriable a GetResult from StreamInput does not + leave metaFields and documentFields null. This could cause an NPE in + situations where upsert response for a document that did not exist is + passed back to a node that forwarded the upsert request. + + closes #48215 + +commit 47e5e34f425a4f636944b192b0bd524b958b80e6 +Author: Tim Vernum +Date: Thu Dec 12 14:37:44 2019 +1100 + + Support "enterprise" license types (#49474) + + This adds "enterprise" as an acceptable type for a license loaded + through the PUT _license API. + + Internally an enterprise license is treated as having a "platinum" + operating mode. + + The handling of License types was refactored to have a new explicit + "LicenseType" enum in addition to the existing "OperatingMode" enum. + + By default (in 7.x) the GET license API will return "platinum" when an + enterprise license is active in order to be compatible with existing + consumers of that API. + A new "accept_enterprise" flag has been introduced to allow clients to + opt-in to receive the correct "enterprise" type. + + Backport of: #49223 + +commit 54467b5d8b0510d6dcf48c909f23decd0d0a8af4 +Author: Ryan Ernst +Date: Wed Dec 11 16:59:21 2019 -0800 + + Simplify running tools in packaging tests (#49665) (#50110) + + Running tools requires a shell. This should be the shell setup by the + base packaging tests, but currently tests must pass in their own shell. + This commit begins to make running tools easier by eliminating the shell + argument, instead keeping the shell as part of the Installation (which + can eventually be passed through from the test itself on installation). + The variable names for each tool are also simplified. + +commit ff6ad583ffcb571ffc6b7f979632bcc31ae03b7b +Author: Ryan Ernst +Date: Mon Dec 9 09:58:44 2019 -0800 + + Make testclusters registry extension name unique (#49956) + + The testclusters registory is a singleton extension element added to the + root project which tracks which test clusters are used throughout the + multi project. But having the same name as the extension used to + configure test clusters within each subprojects breaks using a single + project for an external plugin. This commit renames the registry + extension to make it unique. + + closes #49787 + +commit fdfb1950fcc9c11b57b25282d1302a9ee18d2b07 +Author: Mark Vieira +Date: Wed Dec 11 15:39:06 2019 -0800 + + Allow individual projects to override test heapdump setting (#50096) + + This commit goes from using a JvmArgumentProvider to using the normal + Test task APIs for passing the `HeapDumpOnOutOfMemoryError` JVM + argument. This makes it simpler for subprojects, such as lang-painless + to override this setting if necessary. + + Closes #49117 + + (cherry picked from commit e97c38ff8e862abdc1d7816c66f9869ed216031f) + +commit 38b67f719eec45714c6469e59cb22eaec439a40a +Author: Tim Brooks +Date: Wed Dec 11 16:24:19 2019 -0700 + + Add int indicating size of transport header (#50085) + + Currently we do not know the size of the transport header (map of + request response headers, features array, and action name). This means + that we must read the entire transport message to dependably act on the + headers. This commit adds an int indicating the size of the transport + headers. With this addition we can act upon the headers prior to reading + the entire message. + +commit e9e2e5fc7123e9da9317fadfa38bc9872949d016 +Author: Andrei Stefan +Date: Wed Dec 11 23:53:03 2019 +0200 + + Have COUNT DISTINCT return 0 instead of NULL for no documents matching. (#50037) + + (cherry picked from commit cb94731e6f41bc51c23e4aab495b64eea731a061) + +commit 4619834b970d99b073550732d1a8d633cb4b20aa +Author: Przemko Robakowski +Date: Wed Dec 11 23:06:05 2019 +0100 + + [7.x] CSV ingest processor (#49509) (#50083) + + * CSV ingest processor (#49509) + + This change adds new ingest processor that breaks line from CSV file into separate fields. + By default it conforms to RFC 4180 but can be tweaked. + + Closes #49113 + +commit 23ab9e020408caf89e250ca1b8b860224585e8ee +Author: Jason Tedor +Date: Wed Dec 11 16:02:31 2019 -0500 + + Require JDK 13 for compilation (#50004) + + We have a long history of advancing the required compiler to the newest + JDK. JDK 13 has been with us for awhile, but we were blocked from + upgrading since Gradle was not compatible with JDK 13. With the + advancement in our project to Gradle 6 which supports JDK 13, we can now + advance our minimum compiler version. This commit updates the minimum + compiler version to JDK 13. + +commit adf5c92f8c7a112d2486a8c53ba67a74153e8e28 +Author: Adrien Grand +Date: Wed Dec 11 22:13:58 2019 +0100 + + Address UUIDTests#testCompression failures. (#50101) + + Those were due to codec randomization. + + Closes #50048 + +commit 48d401f97ecd110c563de5b5758aa369fcf32a7f +Author: Mark Vieira +Date: Wed Dec 11 11:04:26 2019 -0800 + + Fix exception message for boolean BuildParams properties (#50094) + +commit 277880bb4fcabe2e4ce9cd02c23cadb4c9aa6570 +Author: Julie Tibshirani +Date: Wed Dec 11 10:33:47 2019 -0800 + + In sparse vector REST tests, specify the index name in searches. (#50061) + + The `sparse_vector` REST tests occasionally fail on 7.x because we don't receive the expected response headers with deprecation warnings. + + One theory as to what is happening is that there is an extra empty index present in addition to the test index. Since the search doesn't specify an index name, it hits both the test index and this extra empty index and shard responses from the extra index don't produce deprecation warnings. If not all shard responses contain the warning headers, then certain deprecation warnings can be lost (due to the bug described in #33936). + + This PR tries to harden the `sparse_vector` tests by always specifying the index name during a search. This doesn't fix the root causes of the issue, but is good practice and can help avoid intermittent failures. + + Addresses #49383. + +commit 03ecaae2219f92b6a9f268f234ba984709d02c8e +Author: Dimitris Athanasiou +Date: Wed Dec 11 18:50:26 2019 +0200 + + [7.x][ML] Avoid classification integ test training on single class (#50072) (#50078) + + The `ClassificationIT.testTwoJobsWithSameRandomizeSeedUseSameTrainingSet` + test was previously set up to just have 10 rows. With `training_percent` + of 50%, only 5 rows will be used for training. There is a good chance that + all 5 rows will be of one class which results to failure. + + This commit increases the rows to 100. Now 50 rows should be used for training + and the chance of failure should be very small. + + Backport of #50072 + +commit d19c8db4e49c29577e988d5ba26af1909e68b718 +Author: Armin Braun +Date: Wed Dec 11 17:40:25 2019 +0100 + + Fix GCS Mock Batch Delete Behavior (#50034) (#50084) + + Batch deletes get a response for every delete request, not just those that actually hit an existing blob. + The fact that we only responded for existing blobs leads to a degenerate response that throws a parse exception if a batch delete only contains non-existant blobs. + +commit 9cdabbd36336e830c2ba42b84577b19ba62c8d0b +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Dec 11 17:18:51 2019 +0100 + + Log attachment generation failures (#50080) + + Watcher logs when actions fail in ActionWrapper, but failures to + generate an email attachment are not logged and we thus only know the + type of the exception and not where/how it occurred. + +commit 285eacd2676b3245fde79436314c5be1d57dca09 +Author: David Turner +Date: Wed Dec 11 15:07:47 2019 +0000 + + Use more specific loggers in subclasses of TMNA (#50076) + + Adjusts the subclasses of `TransportMasterNodeAction` to use their own loggers + instead of the one for the base class. + + Relates #50056. + Partial backport of #46431 to 7.x. + +commit 8ff809af2d3b3c43371a30df57b5cb85fd460222 +Author: Wilder Pereira +Date: Wed Dec 11 11:55:53 2019 -0300 + + [DOCS] Replace interval notation with plain English in match query docs (#47334) + + As we discussed in #36371, interval notation is confusing to some users. This makes the intention clearer by just explaining inclusivity and exclusivity in the docs. + +commit 9b116c8fefb94905112f7a314c146e5e5117807a +Author: Przemysław Witek +Date: Wed Dec 11 09:35:05 2019 +0100 + + A few improvements to AnalyticsProcessManager class that make the code more readable. (#50026) (#50069) + +commit 3b613c36f4da6a3c03ba5caebb68d26ae31a3ba6 +Author: Ioannis Kakavas +Date: Wed Dec 11 09:14:50 2019 +0200 + + Always return 401 for not valid tokens (#49736) (#50042) + + Return a 401 in all cases when a request is submitted with an + access token that we can't consume. Before this change, we would + throw a 500 when a request came in with an access token that we + had generated but was then invalidated/expired and deleted from + the tokens index. + + Resolves: #38866 + Backport of #49736 + +commit 44cd2f444c04dd575e0c3a05b1082c5c231ab29f +Author: Stuart Cam +Date: Wed Dec 11 13:23:00 2019 +1100 + + Add the REST API specifications for SLM Status / Start / Stop endpoints. (#49759) + + Was originally missed in PR #47710 + + (cherry picked from commit 133b34c8355639ae0f699a86ffd9f37d19f73bca) + +commit eb20db8a1c429abf92ee7599fc7f8c8786bb787e +Author: Jack Conradson +Date: Tue Dec 10 12:55:06 2019 -0800 + + Update Painless AST Catch Node (#50044) + + This makes two changes to the catch node: + + 1. Use SDeclaration to replace independent variable usage. + 2. Use a DType to set a "minimum" exception type - this allows us to require + users to continue using Exception as "minimum" type for catch blocks, but + for us to internally catch Error/Throwable. This is a required step to + removing custom try/catch blocks from SClass. + +commit ba9526ec4f30f471f415d2a8c419158738a23427 +Author: William Brafford +Date: Tue Dec 10 15:00:05 2019 -0500 + + Refactor utility code in qa:os: tests (#49945) (#50000) + + This refactor bridges some gaps between a long-running feature branch (#49268) and the master branch. + + First of all, this PR gives our PackagingTestCase class some methods to start and stop Elasticsearch that will switch on packaging type and delegate to the appropriate utility class for deb/RPM packages, archive installations, and Docker. These methods should be very useful as we continue group tests by function rather than by package or platform type. + + Second, the password-protected keystore tests have a particular need to read the output of Elasticsearch startup commands. In order to make this easer to do, some commands now return Shell.Result objects so that tests can check over output to the shell. To that end, there's also an assertElasticsearchFailure method that will handle checking for startup failures for the various distribution types. + + There is an update to the Powershell startup script for archives that asynchronously redirects the output of the Powershell process to files that we can read for errors. + + Finally, we use the ES_STARTUP_SLEEP_TIME environment variable to make sure that our startup commands wait long enough before exiting for errors to make it to the standard output and error streams. + +commit df558aa0ca6750f3cf43b16114f9bb8304bbf556 +Author: Patryk Krawaczyński +Date: Tue Dec 10 20:23:14 2019 +0100 + + [DOCS] Document `index.queries.cache.enabled` as a static setting (#49886) + +commit 87e72156ce54aa25b7dc61b45d6ccbf45a2fe07c +Author: Adrien Grand +Date: Tue Dec 10 18:04:58 2019 +0100 + + Upgrade to lucene 8.4.0-snapshot-662c455. (#50016) (#50039) + + Lucene 8.4 is about to be released so we should check it doesn't cause problems + with Elasticsearch. + +commit 1a6e5bf220cec1a2de55f2638f6b4efc876389c4 +Author: Peter Johnson +Date: Tue Dec 10 17:32:42 2019 +0100 + + [Docs] Fix typo in function-score-query.asciidoc (#50030) + +commit 15f27d8c5422fd80534c0969db3c371b74881d16 +Author: Lisa Cawley +Date: Tue Dec 10 08:03:43 2019 -0800 + + [DOCS] Removes realm type security setting (#50001) + +commit 81ff2d0f0dd6fdf35001e04a1660e2a7b3c695b1 +Author: Przemyslaw Gomulka +Date: Tue Dec 10 16:43:41 2019 +0100 + + Allow skipping ranges of versions backport(#50014) (#50028) + + Multiple version ranges are allowed to be used in section skip in yml + tests. This is useful when a bugfix was backported to latest versions + and all previous releases contain a wire breaking bug. + examples: + 6.1.0 - 6.3.0, 6.6.0 - 6.7.9, 7.0 - + - 7.2, 8.0.0 - + backport #50014 + +commit 3f5678ca7974bab78639e197664c8873570a36be +Author: James Rodewig +Date: Tue Dec 10 09:30:04 2019 -0500 + + [DOCS] Remove shadow replica reference (#50029) + + Removes a reference to shadow replicas from the cat shards API docs + and a comment in cluster/routing/UnassignedInfo.java. + + Shadow replicas were removed with #23906. + +commit 8891f4db88b1dbdea310bf1fe33e771f7950ad1d +Author: Dimitris Athanasiou +Date: Tue Dec 10 15:29:19 2019 +0200 + + [7.x][ML] Introduce randomize_seed setting for regression and classification (#49990) (#50023) + + This adds a new `randomize_seed` for regression and classification. + When not explicitly set, the seed is randomly generated. One can + reuse the seed in a similar job in order to ensure the same docs + are picked for training. + + Backport of #49990 + +commit ee4a8a08dd5bc8e5642e3e8946935b73dca4e719 +Author: Armin Braun +Date: Tue Dec 10 13:01:51 2019 +0100 + + Improve Snapshot Finalization Ex. Handling (#49995) (#50017) + + * Improve Snapshot Finalization Ex. Handling + + Like in #49989 we can get into a situation where the setting of + the repository generation (during snapshot finalization) in the cluster + state fails due to master failing over. + In this case we should not try to execute the next cluster state update + that will remove the snapshot from the cluster state. + + Closes #49989 + +commit 3d8c2f9e182b68aa3724257865a8c7bc8b658ece +Author: Alan Woodward +Date: Tue Dec 10 10:44:31 2019 +0000 + + Fix query analyzer logic for mixed conjunctions of terms and ranges (#49803) + + When the query analyzer examines a conjunction containing both terms and ranges, + it should only include ranges in the minimum_should_match calculation if there are no + other range queries on that same field within the conjunction. This is because we cannot + build a selection query over disjoint ranges on the same field, and it is not easy to check + if two range queries have an overlap. + + The current logic to calculate this just sets minimum_should_match to 1 or 0, dependent + on whether or not the current range is over a field that has already been seen. However, this + can be incorrect in the case that there are terms in the same match group which adjust the + minimum_should_match downwards. Instead, the logic should be changed to match the + terms extraction, whereby we adjust minimum_should_match downwards if we have already + seen a range field. + + Fixes #49684 + +commit a16abf921fa9795f7b869dae3977373bcfa02260 +Author: Yannick Welsch +Date: Tue Dec 10 09:45:27 2019 +0100 + + Make elasticsearch-node tools custom metadata-aware (#48390) + + The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently + unaware of plugins and will therefore drop custom metadata from the cluster state once the + state is written out again (as it skips over the custom metadata that it can't read). This commit + preserves unknown customs when editing on-disk metadata through the elasticsearch-node + command-line tools. + +commit d3cf89b563301a42f96fc4d2dd8f3a32cb27cb96 +Author: Mark Vieira +Date: Mon Dec 9 11:34:35 2019 -0800 + + Upgrade to Gradle 6.0 (#49211) (#49994) + + This upgrade required a few significant changes. Firstly, the build + scan plugin has been renamed, and changed to be a Settings plugin rather + than a project plugin so the declaration of this has moved to our + settings.gradle file. Second, we were using a rather old version of the + Nebula ospackage plugin for building deb and rpm packages, the migration + to the latest version required some updates to get things working as + expected as we had some workarounds in place that are no longer + applicable with the latest bug fixes. + + (cherry picked from commit 87f9c16e2f8870e3091062cde37b43042c3ae1c5) + +commit 33594380c7f779ab5fb3b0718cc1cd7c1e2f6b3b +Author: James Rodewig +Date: Mon Dec 9 13:16:16 2019 -0500 + + [DOCS] Skip synced flush docs tests (#49986) + + The current snippets in the synced flush docs can cause conflicts with + other background syncs, such as the global checkpoint sync or retention + lease sync, in the docs tests. + + This skips tests for those snippets to avoid conflicts. + +commit dd441962bb5c1958ec9f2fc616bc6e56f73d3ee7 +Author: shiwenjie12 <656336863@qq.com> +Date: Tue Dec 10 02:02:55 2019 +0800 + + Modify notes (#48331) + + Modify notes + +commit a9d977775df719ef36dca2b1cea7f9af43cc3e26 +Author: Ryan Ernst +Date: Mon Dec 9 09:57:29 2019 -0800 + + Remove leftover debug log message (#49957) + + This was leftover from debugging #49204. + +commit bfb2dc13539ec27f6305744248e9718521e08efa +Author: Jason Tedor +Date: Mon Dec 9 12:39:26 2019 -0500 + + Enable dependent settings values to be validated (#49942) + + Today settings can declare dependencies on another setting. This + declaration is implemented so that if the declared setting is not set + when the declaring setting is, settings validation fails. Yet, in some + cases we want not only that the setting is set, but that it also has a + specific value. For example, with the monitoring exporter settings, if + xpack.monitoring.exporters.my_exporter.host is set, we not only want + that xpack.monitoring.exporters.my_exporter.type is set, but that it is + also set to local. This commit extends the settings infrastructure so + that this declaration is possible. The use of this in the monitoring + exporter settings will be implemented in a follow-up. + +commit 48e74203072182c59020f65e4777324abfc00fec +Author: Marios Trivyzas +Date: Mon Dec 9 15:34:23 2019 +0100 + + SQL: [Tests] Unmute Pivot from NodeSublassTests (#49925) + + The `testReplaceChildren()` has been fixed for Pivot as part + of #49693. + + Reverting: #49045 + (cherry picked from commit 4b9b9edbcf2041a8619b65580bbe192bf424cebc) + +commit 322dabe3de523fb4bf65b2ae3b3a77c2466e5908 +Author: James Rodewig +Date: Mon Dec 9 11:04:20 2019 -0500 + + [DOCS] Correct `for in` example in Painless docs (#49991) + + Adds a needed `def` keyword to the `for in` example in the Painless docs. + +commit d073bccaad22972c04aa8f8b831adc5140e10158 +Author: Artur Carvalho +Date: Mon Dec 9 16:24:03 2019 +0100 + + [Docs] Fix typo in getting-started.asciidoc (#49985) + +commit 0b6ce9683cf12f4c171cfcae5cf1746c5cac35df +Author: Benjamin Trent +Date: Mon Dec 9 10:14:41 2019 -0500 + + [ML] Use query in cardinality check (#49939) (#49984) + + When checking the cardinality of a field, the query should be take into account. The user might know about some bad data in their index and want to filter down to the target_field values they care about. + +commit 056c698540b1bb51eb09f28b4e51e7be7e98dbbc +Author: Vishnu Chilamakuru +Date: Mon Dec 9 19:09:05 2019 +0530 + + Add Validation for maxQueryTerms to be greater than 0 for MoreLikeThisQuery (#49966) + + Adds validation for maxQueryTerms to be greater than 0 for MoreLikeThisQuery + and MoreLikeThisQueryBuilder. + + Closes #49927 + +commit 1918a21baf4ce1c397fdf6792da9a9ce25c920f6 +Author: James Rodewig +Date: Mon Dec 9 08:39:17 2019 -0500 + + [DOCS] Correct inline shape snippets in shape query docs (#49921) + + In the shape query docs, the index mapping snippet uses the "geometry" + shape field mapping. However, the doc index snippet uses the "location" + property. + + This changes the "location" property to "geometry". It also adds a + comment containing the search result snippet. This should prevent + similar issues in the future. + +commit 0965a10468a8982ce973f801477709775df4064a +Author: Przemysław Witek +Date: Mon Dec 9 14:43:01 2019 +0100 + + [7.x] Pass `prediction_field_type` to C++ analytics process (#49861) (#49981) + +commit 049d854360f8263a4ca57bdaf7b571beea696739 +Author: Benjamin Trent +Date: Mon Dec 9 08:29:45 2019 -0500 + + [ML][Inference] adjust so target_field always has inference result and optionally allow new top classes field in the classification config (#49923) (#49982) + +commit 62e128f02d49819f83ce201663c7b8be7a8ea69f +Author: Armin Braun +Date: Mon Dec 9 12:05:55 2019 +0100 + + Cleanup Old index-N Blobs in Repository Cleanup (#49862) (#49902) + + * Cleanup Old index-N Blobs in Repository Cleanup + + Repository cleanup didn't deal with old index-N, this change adds + cleaning up all old index-N found in the repository. + +commit e4f838e764ba6cfadb3a8a0c2ced24cf6339cab1 +Author: Dimitris Athanasiou +Date: Mon Dec 9 11:52:06 2019 +0200 + + [7.x][ML] Update expected mem estimate in explain API integ test (#49924) (#49979) + + Work in progress in the c++ side is increasing memory estimates + a bit and this test fails. At the time of this commit the mem + estimate when there is no source query is a about 2Mb. So I + am relaxing the test to assert memory estimate is less than 1Mb + instead of 500Kb. + + Backport of #49924 + +commit 549b103458b8c39d446a6ea79d2a65e6aba6aacc +Author: cachedout +Date: Mon Dec 9 08:25:03 2019 +0000 + + [7.x] APM system_user (#47668) (#49912) + + * Add test for APM beats index perms + + * Grant monitoring index privs to apm_system user + + * Review feedback + + * Fix compilation problem + +commit ac2774c9facc05d6fb2b785ae20a0c307e8fc0bf +Author: Armin Braun +Date: Mon Dec 9 09:02:57 2019 +0100 + + Use Cluster State to Track Repository Generation (#49729) (#49976) + + Step on the road to #49060. + + This commit adds the logic to keep track of a repository's generation + across repository operations. See changes to package level Javadoc for the concrete changes in the distributed state machine. + + It updates the write side of new repository generations to be fully consistent via the cluster state. With this change, no `index-N` will be overwritten for the same repository ever. So eventual consistency issues around conflicting updates to the same `index-N` are not a possibility any longer. + + With this change the read side will still use listing of repository contents instead of relying solely on the cluster state contents. + The logic for that will be introduced in #49060. This retains the ability to externally delete the contents of a repository and continue using it afterwards for the time being. In #49060 the use of listing to determine the repository generation will be removed in all cases (except for full-cluster restart) as the last step in this effort. + +commit 7a2e35caa0589371060d256badd054090cc13388 +Author: Yannick Welsch +Date: Mon Dec 9 08:28:23 2019 +0100 + + Properly fake corrupted translog (#49918) + + The fake translog corruption in the test sometimes generates invalid translog files where some + assertions do not hold (e.g. minSeqNo <= maxSeqNo or minTranslogGen <= translogGen) + + Closes #49909 + +commit 01d36afa4b920f44bba678ca341995cf8f187c2e +Author: Yannick Welsch +Date: Mon Dec 9 08:27:41 2019 +0100 + + Randomly run CCR tests with _source disabled (#49922) + + Makes sure that CCR also properly works with _source disabled. + + Changes one exception in LuceneChangesSnapshot as the case of missing _recovery_source + because of a missing lease was not properly properly bubbled up to CCR (testIndexFallBehind + was failing). + +commit f768f8ddab4996d2fc5ccfd7e6c6bdffdfb8da43 +Author: Armin Braun +Date: Sun Dec 8 17:36:05 2019 +0100 + + Fix TimedRunnable Executing onAfter Twice (#49910) (#49930) + + If we have a nested `AbstractRunnable` inside of `TimedRunnable` + it's executed twice on `run` (once when its own `run` method is invoked and once when + the `onAfter` in the `TimedRunnable` is executed). + Simply removing the `onAfter` override in `TimedRunnable` makes sure that the `onAfter` + is only called once by the `run` on the nested `AbstractRunnable` itself. + Same was done for `onFailure` as it was double-triggering as well on exceptions in the inner `onFailure`. + +commit 8ae11e176a95d7dbc4c854f3a2df6edc22ee7099 +Author: Armin Braun +Date: Sun Dec 8 16:14:20 2019 +0100 + + Cleanup some in o.e.transport (#49901) (#49971) + + Cleaning up some obvious compile warnings and dead code. + +commit 5b896c5bb5892cb4831b8abb06303983c5d5449d +Author: Costin Leau +Date: Sat Dec 7 11:02:14 2019 +0200 + + SQL: Refactor usage of NamedExpression (#49693) + + To recap, Attributes form the properties of a derived table. + Each LogicalPlan has Attributes as output since each one can be part of + a query and as such its result are sent to its consumer. + This change essentially removes the name id comparison so any changes + applied to existing expressions should work as long as the said + expressions are semantically equivalent. + This change enforces the hashCode and equals which has the side-effect + of using hashCode as identifiers for each expression. + By removing any property from an Attribute, the various components need + to look the original source for comparison which, while annoying, should + prevent a reference from getting out of sync with its source due to + optimizations. + + Essentially going forward there are only 3 types of NamedExpressions: + + Alias - user define (implicit or explicit) name + FieldAttribute - field backed by Elasticsearch + ReferenceAttribute - a reference to another source acting as an + Attribute. Typically the Attribute of an Alias. + + * Remove the usage of NamedExpression as basis for all Expressions. + Instead, restrict their use only for named context, such as projections + by using Aliasing instead. + * Remove different types of Attributes and allow only FieldAttribute, + UnresolvedAttribute and ReferenceAttribute. To avoid issues with + rewrites, resolve the references inside the QueryContainer so the + information always stays on the source. + * Side-effect, simplify the rules as the state for InnerAggs doesn't + have to be contained anymore. + * Improve ResolveMissingRef rule to handle references to named + non-singular expression tree against the same expression used up the + tree. + + #49693 backport to 7.x + + (cherry picked from commit 5d095e2173bcbf120f534a6f2a584185a7879b57) + +commit e66cfc4369997224365711cb2fd6ddc3aeccbcd0 +Author: Ryan Ernst +Date: Fri Dec 6 17:43:12 2019 -0800 + + Fix incorrect use of multiline NOTE in rpm docs (#49962) + + This was a copy/paste error from #49893. This commit converts the NOTE + to use inline style instead of one needing closing linebreak. + +commit d29f04209b0b3e3065b4f04e98e5d6ce2485019d +Author: Ryan Ernst +Date: Fri Dec 6 15:54:30 2019 -0800 + + Disable repo configuration for rpm based systems (#49893) + + This commit changes the recommended repository file for rpm based + systems to be disabled by default. This is a safer practice so upgrades + of the system do no accidentally upgrade elasticsearch itself. + + closes #30660 + +commit 401c75d8b5d8eec4275de9126ac1c0b6362d1c4a +Author: Ryan Ernst +Date: Thu Dec 5 17:26:35 2019 -0800 + + Dump wildfly log on start failure (#49892) + + When testing wildfly with Elasticsearch, we currently dump the wildfly + log if the test fails. However, when starting wildfly we may fail to + find the port number wildfly started on, and fail with no output. This + change dumps the wildflog log when failing to find the http or + management ports. + + relates #49374 + +commit d7083a84f4b82ceb6d3f289de9e280239998a380 +Author: Przemko Robakowski +Date: Sat Dec 7 00:19:09 2019 +0100 + + Allow list of IPs in geoip ingest processor (#49573) (#49947) + + * Allow list of IPs in geoip ingest processor + + This change lets you use array of IPs in addition to string in geoip processor source field. + It will set array containing geoip data for each element in source, unless first_only parameter + option is enabled, then only first found will be returned. + + Closes #46193 + +commit 17cda5b2c0480ce062f6ffad4aa41b5e9a5b52b1 +Author: Stuart Tettemer +Date: Fri Dec 6 15:08:05 2019 -0700 + + Scripting: Groundwork for caching script results (#49895) (#49944) + + In order to cache script results in the query shard cache, we need to + check if scripts are deterministic. This change adds a default method + to the script factories, `isResultDeterministic() -> false` which is + used by the `QueryShardContext`. + + Script results were never cached and that does not change here. Future + changes will implement this method based on whether the results of the + scripts are deterministic or not and therefore cacheable. + + Refs: #49466 + + **Backport** + +commit 8205cdd423a68f55891a18f0b1c35d1944e1ac73 +Author: Lee Hinman +Date: Fri Dec 6 12:55:16 2019 -0700 + + [7.x] Refactor IndexLifecycleRunner to split state modificatio… (#49936) + + This commit refactors the `IndexLifecycleRunner` to split out and + consolidate the number of methods that change state from within ILM. It + adds a new class `IndexLifecycleTransition` that contains a number of + static methods used to modify ILM's state. These methods all return new + cluster states rather than making changes themselves (they can be + thought of as helpers for modifying ILM state). + + Rather than having multiple ways to move an index to a particular step + (like `moveClusterStateToStep`, `moveClusterStateToNextStep`, + `moveClusterStateToPreviouslyFailedStep`, etc (there are others)) this + now consolidates those into three with (hopefully) useful names: + + - `moveClusterStateToStep` + - `moveClusterStateToErrorStep` + - `moveClusterStateToPreviouslyFailedStep` + + In the move, I was also able to consolidate duplicate or redundant + arguments to these functions. Prior to this commit there were many calls + that provided duplicate information (both `IndexMetaData` and + `LifecycleExecutionState` for example) where the duplicate argument + could be derived from a previous argument with no problems. + + With this split, `IndexLifecycleRunner` now contains the methods used to + actually run steps as well as the methods that kick off cluster state + updates for state transitions. `IndexLifecycleTransition` contains only + the helpers for constructing new states from given scenarios. + + This also adds Javadocs to all methods in both `IndexLifecycleRunner` + and `IndexLifecycleTransition` (this accounts for almost all of the + increase in code lines for this commit). It also makes all methods be as + restrictive in visibility, to limit the scope of where they are used. + + This refactoring is part of work towards capturing actions and + transitions that ILM makes, by consolidating and simplifying the places + we make state changes, it will make adding operation auditing easier. + +commit 1c5a139968a821fd3f56a30cb498fe2b84e080ad +Author: Jake Landis +Date: Fri Dec 6 13:39:33 2019 -0600 + + Update jackson-databind to 2.8.11.4 (#49347) (#49937) + +commit 17fa9d584485ad1db627a0494ca781a44a435964 +Author: David Roberts +Date: Fri Dec 6 17:04:09 2019 +0000 + + [TEST] Mute ConnectionManagerTests.testConcurrentConnectsAndDisconnects + + Due to https://github.com/elastic/elasticsearch/issues/49903 + +commit d299bf576033d41429ce365b32ade730361e1bd9 +Author: Alexander Reelsen +Date: Fri Dec 6 14:33:02 2019 +0100 + + Add tests for ingesting CBOR data attachments (#49715) + + Our docs specifically mention that CBOR is supported when ingesting attachments. However this is not tested anywhere. + + This adds a test, that uses specifically CBOR format in its IndexRequest and another one that behaves like CBOR in the ingest attachment unit tests. + +commit e60837aa3bd17aecceda40a41fcdc82afbe88e5f +Author: Przemysław Witek +Date: Fri Dec 6 14:31:17 2019 +0100 + + [7.x] Log whole analytics stats when the state assertion fails (#49906) (#49911) + +commit 63d3933787a297c44ca5527555fff31a392feb9a +Author: István Zoltán Szabó +Date: Fri Dec 6 13:24:22 2019 +0100 + + [DOCS] Fixes classification evaluation example response. (#49905) + +commit bb91291273f6c89426d43174177c0dce8d8cadb9 +Author: István Zoltán Szabó +Date: Fri Dec 6 10:23:01 2019 +0100 + + [DOCS] Fixes attribute in transforms overview. (#49898) + +commit 1d3feaf18eaef1aaa11b97b8d2aae114a25d61f5 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Dec 6 09:44:36 2019 +0100 + + Reindex sort deprecation warning take 2 (#49855) (#49899) + + Moved the deprecation warning to ReindexValidator to ensure it runs + early and works with resilient reindex. Also check that the warning + is reported back for wait_for_completion=false. + + Follow-up to #49458 + +commit b17cfc93e3164265a63a8b90e23b512888192f6f +Author: Hendrik Muhs +Date: Fri Dec 6 08:19:21 2019 +0100 + + [Transform][DOCS]rewrite client ip example to use continuous transform (#49822) + + adapt the transform example for suspicious client ips to use continuous transform + +commit cd3744c0b7640f3b2d7c3b9cd225651b93e9247c +Author: Jack Conradson +Date: Thu Dec 5 16:57:45 2019 -0800 + + Add nodes to handle types (#49785) + + This PR adds 3 nodes to handle types defined by a front-end creating a + Painless AST. These types are decided with data immutability in mind - + hence the reason for more than a single node. + +commit abd6fa149c78efe12cdc21a94a756c6f2028cdfb +Author: Mark Vieira +Date: Thu Dec 5 16:31:07 2019 -0800 + + Move BuildParams class to 'minimumRuntime' source set (#49890) + + Move BuildParams class to 'minimumRuntime' source set to retain compatibility + with build-tools for builds using a Java 8 runtime. + + Closes #49766 + + (cherry picked from commit 1059f823acdfa7a2f1f9bff21c7256dae4f3e23c) + +commit 0f02e02d7735a7d4f9f45bcb796d0ccfe8af6e22 +Author: Orhan Toy +Date: Thu Dec 5 22:33:36 2019 +0100 + + Consistent case in CLI option descriptions (#49635) + + This commit improves the casing of messages in the CLI help descriptions. + +commit fec882a45781fd4216f1012b6edae80f5947f6cc +Author: Zachary Tong +Date: Thu Dec 5 16:11:27 2019 -0500 + + Decouple pipeline reductions from final agg reduction (#45796) + + Historically only two things happened in the final reduction: + empty buckets were filled, and pipeline aggs were reduced (since it + was the final reduction, this was safe). Usage of the final reduction + is growing however. Auto-date-histo might need to perform + many reductions on final-reduce to merge down buckets, CCS + may need to side-step the final reduction if sending to a + different cluster, etc + + Having pipelines generate their output in the final reduce was + convenient, but is becoming increasingly difficult to manage + as the rest of the agg framework advances. + + This commit decouples pipeline aggs from the final reduction by + introducing a new "top level" reduce, which should be called + at the beginning of the reduce cycle (e.g. from the SearchPhaseController). + This will only reduce pipeline aggs on the final reduce after + the non-pipeline agg tree has been fully reduced. + + By separating pipeline reduction into their own set of methods, + aggregations are free to use the final reduction for whatever + purpose without worrying about generating pipeline results + which are non-reducible + +commit 721a8b3d9c1321f854b4c26fd75eace7aa9f94fb +Author: Ryan Ernst +Date: Thu Dec 5 13:03:06 2019 -0800 + + Fix external integ test zip dep to expect a zip (#49813) + + When external plugin authors use build-tools, their integ tests depend + on the integ-test-zip artifact. However, this dependency was broken in + 7.5.0 by accidentally removing the `@zip` qualifier on the maven + dependency, which works around the fact the pom for the integ-test-zip + claims the artifact is a pom instead of zip packaging. This commit + restores the workaround of using `@zip` until the pom can be fixed. + + closes #49787 + +commit 687c6648d99877ed0063e2cb317c989025d07b7a +Author: Jack Conradson +Date: Thu Dec 5 12:08:23 2019 -0800 + + Minor Painless Clean Up (#49844) + + This cleans up two minor things. + - Cleans up style of == false + - Pulls maxLoopCounter into a member variable instead of accessing + CompilerSettings multiple times in the SFunction node + +commit 1641fcd488d303bbc4677ed1ac98abbeeddb3255 +Author: Orhan Toy +Date: Thu Dec 5 20:24:22 2019 +0100 + + [DOCS] Minor typo fixes in reindex.asciidoc (#49863) + +commit 1d97cee3151f74b94689e6c21a6f5c4eab643a07 +Author: James Baiera +Date: Thu Dec 5 13:53:42 2019 -0500 + + Support es7 node http publish_address format (#49279) (#49839) + + Add parsing support to node http publish_address format cname/ip:port. + +commit b281d64e89fac6e088708c539f6ed89783f247db +Author: Tim Brooks +Date: Thu Dec 5 10:39:57 2019 -0700 + + Ensure remote strategy settings can be updated (#49812) + + This is related to #49067. As part of this work a new sniff number of + node connections setting, a simple addresses setting, and a simple + number of sockets setting have been added. This commit ensures that + these settings are properly hooked up to support dynamic updates. + +commit f4b3bb7d6b17b9d443db326436ce96f1052356c2 +Author: István Zoltán Szabó +Date: Thu Dec 5 14:15:19 2019 +0100 + + [DOCS] Adds an example of preprocessing actions to the PUT DFA API docs (#49831) + +commit 495762486d8755dfa96a05105b9aeb0d7406714f +Author: Jim Ferenczi +Date: Wed Dec 4 16:50:15 2019 +0100 + + Fix concurrent issue in SearchPhaseController (#49829) + + The list used by the search progress listener can be nullified + by another thread that reports a query result. This change replaces + the usage of this list with a new array that is synchronously modified. + + Closes #49778 + +commit 04e99ff1ee2d8e5c56412512bfbd26374625f9e6 +Author: István Zoltán Szabó +Date: Thu Dec 5 09:57:01 2019 +0100 + + [DOCS] Fixes typo in the ML anomaly detection time functions docs. (#49834) + +commit 426c7a5e8f8c4cb8d6e907e3deeeebadeb353394 +Author: Stuart Tettemer +Date: Wed Dec 4 16:18:22 2019 -0700 + + Scripting: add available languages & contexts API (#49652) (#49815) + + Adds `GET /_script_language` to support Kibana dynamic scripting + language selection. + + Response contains whether `inline` and/or `stored` scripts are + enabled as determined by the `script.allowed_types` settings. + + For each scripting language registered, such as `painless`, + `expression`, `mustache` or custom, available contexts for the language + are included as determined by the `script.allowed_contexts` setting. + + Response format: + ``` + { + "types_allowed": [ + "inline", + "stored" + ], + "language_contexts": [ + { + "language": "expression", + "contexts": [ + "aggregation_selector", + "aggs" + ... + ] + }, + { + "language": "painless", + "contexts": [ + "aggregation_selector", + "aggs", + "aggs_combine", + ... + ] + } + ... + ] + } + ``` + + Fixes: #49463 + + **Backport** + +commit dbf6183469102a8b8d4104b0b427a7488947b78d +Author: Jack Conradson +Date: Wed Dec 4 11:43:55 2019 -0800 + + Remove extraneous pass (#49797) + + This removes the storeSettings pass where nodes in the AST could store + information they needed out of CompilerSettings for use during later + passes. CompilerSettings is part of ScriptRoot which is available during the + analysis pass making the storeSettings pass redundant. + +commit ce2ca3bd3db231163e25cbf282a256411c95df81 +Author: Ryan Ernst +Date: Wed Dec 4 11:56:37 2019 -0800 + + Fix task input for docker build (#49814) + + The docker build task depends on the docker context being built, but it + was not explicitly setup as an input. This commit adds the task as an + input to the docker build. + + relates #49613 + +commit 91ac87d75b155c6f61792103c197c38e2a817523 +Author: Armin Braun +Date: Wed Dec 4 19:36:52 2019 +0100 + + Stop Allocating Buffers in CopyBytesSocketChannel (#49825) (#49832) + + * Stop Allocating Buffers in CopyBytesSocketChannel (#49825) + + The way things currently work, we read up to 1M from the channel + and then potentially force all of it into the `ByteBuf` passed + by Netty. Since that `ByteBuf` tends to by default be `64k` in size, + large reads will force the buffer to grow, completely circumventing + the logic of `allocHandle`. + + This seems like it could break + `io.netty.channel.RecvByteBufAllocator.Handle#continueReading` + since that method for the fixed-size allocator does check + whether the last read was equal to the attempted read size. + So if we set `64k` because that's what the buffer size is, + then wirte `1M` to the buffer we will stop reading on the IO loop, + even though the channel may still have bytes that we can read right away. + + More imporatantly though, this can lead to running OOM quite easily + under IO pressure as we are forcing the heap buffers passed to the read + to `reallocate`. + + Closes #49699 + +commit 42f902977dcb50cd70c093cecb96213175005f86 +Author: James Rodewig +Date: Wed Dec 4 12:44:13 2019 -0500 + + [DOCS] Document `minimum_should_match` defaults for `bool` query (#48865) + + Adds documentation for the `minimum_should_match` parameter to the `bool` query docs. Includes docs for the default values: + + - `1` if the `bool` query includes at least one `should` clause and no `must` or `filter` clauses + - `0` otherwise + +commit e3c959b7f1f86db42e78a1bf81910c18b61e4eee +Author: Dimitris Athanasiou +Date: Wed Dec 4 18:49:08 2019 +0200 + + [7.x][ML][HLRC] DF analytics setVersion and setCreateTime should not be public (#49826) (#49833) + + `version` and `create_time` are assigned from the action itself and thus should not + be able to be set from the client. + + Backport of #49826 + +commit 87a73b6bdff440897165c3a4944b8f34da0f4261 +Author: James Rodewig +Date: Wed Dec 4 09:58:19 2019 -0500 + + [DOCS] Reformat length token filter docs (#49805) + + * Adds a title abbreviation + * Updates the description and adds a Lucene link + * Reformats the parameters section + * Adds analyze, custom analyzer, and custom filter snippets + + Relates to #44726. + +commit 6dcb7fa50eede002d826a137999302f3e7b83fea +Author: Yannick Welsch +Date: Wed Dec 4 13:49:43 2019 +0100 + + Add SecureSM support for newer IDEA versions (#49747) + + IntelliJ IDEA moved their JUnit runner to a different package. While this does not break running + tests in IDEA, it leads to an ugly exception being thrown at the end of the tests: + + Exception in thread "main" java.lang.SecurityException: java.lang.System#exit(0) calls are not + allowed + at org.elasticsearch.secure_sm.SecureSM$2.run(SecureSM.java:248) + at org.elasticsearch.secure_sm.SecureSM$2.run(SecureSM.java:215) + at java.base/java.security.AccessController.doPrivileged(AccessController.java:310) + at org.elasticsearch.secure_sm.SecureSM.innerCheckExit(SecureSM.java:215) + at org.elasticsearch.secure_sm.SecureSM.checkExit(SecureSM.java:206) + at java.base/java.lang.Runtime.exit(Runtime.java:111) + at java.base/java.lang.System.exit(System.java:1781) + at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:59) + + This commit adds support for newer IDEA versions in SecureSM. + +commit aa443c6362746834ff484cdcbd76c691b6cd2022 +Author: Alan Woodward +Date: Wed Dec 4 12:16:36 2019 +0000 + + [CI] Interval queries cannot be cached if they use scripts (#49824) + + not adjust testCacheability(), which how fails occasionally when given a random + interval source containing a script. This commit overrides testCacheability() to + explicitly sources with and without script filters. + + Fixes #49821 + +commit 312190266ef0769757953f1d3a423650fb97ca34 +Author: Alan Woodward +Date: Wed Dec 4 11:32:32 2019 +0000 + + Improve coverage of equals/hashCode tests for IntervalQueryBuilder (#49820) + + By default, AbstractQueryTestCase only changes name and boost in its mutateInstance + method, used when checking equals and hashcode implementations. This commit adds + a mutateInstance method to InveralQueryBuilderTests that will check hashcode and + equality when the field or intervals source are changed. + +commit 53d801c0d71adacf5f1f582b6fc31f07240e805e +Author: jimczi +Date: Wed Dec 4 12:21:02 2019 +0100 + + \#49566 Fix non-deterministic sort order in testHighlightingWithKeywordIgnoreBoundaryScanner + +commit 44e94555eec7879fb190f0c5616463590c107a2f +Author: Ignacio Vera +Date: Wed Dec 4 11:51:53 2019 +0100 + + Add reusable HistogramValue object (#49799) (#49823) + + Adds a reusable implementation of HistogramValue so we do not create + an object per document. + +commit 1d522c660551a8b51c6e0eb275ff0e2d4a115e61 +Author: jimczi +Date: Wed Dec 4 11:25:47 2019 +0100 + + add missing change after backport of #49566 + +commit 691421f287273d67331f646351d015bbf1f91fb3 +Author: Jim Ferenczi +Date: Wed Dec 4 11:13:32 2019 +0100 + + Fix invalid break iterator highlighting on keyword field (#49566) + + By default the unified highlighter splits the input into passages using + a sentence break iterator. However we don't check if the field is tokenized + or not so `keyword` field also applies the break iterator even though they can + only match on the entire content. This means that by default we'll split the + content of a `keyword` field on sentence break if the requested number of fragments + is set to a value different than 0 (default to 5). This commit changes this behavior + to ignore the break iterator on non-tokenized fields (keyword) in order to always + highlight the entire values. The number of requested fragments control the number of + matched values are returned but the boundary_scanner_type is now ignored. + Note that this is the behavior in 6x but some refactoring of the Lucene's highlighter + exposed this bug in Elasticsearch 7x. + +commit 1bc3e69fa31857ba52bbc556e048c8a034469cb1 +Author: Rory Hunter +Date: Wed Dec 4 08:57:58 2019 +0000 + + Migrate some of the Docker tests from old repository (#49792) + + Backport of #49079. Reimplement a number of the tests from + elastic/elasticsearch-docker. + + There is also one Docker image fix here, which is that two of the provided + config files had different file permissions to the rest. I've fixed this + with another RUN chmod while building the image, and adjusted the + corresponding packaging test. + +commit 408f25e0169316e78fd4928cf01e3853b369a8b9 +Author: Alan Woodward +Date: Wed Dec 4 08:47:40 2019 +0000 + + Fixes a bug in interval filter serialization (#49793) + + There is a possible NPE in IntervalFilter xcontent serialization when scripts are + used, and `equals` and `hashCode` are also incorrectly implemented for script + filters. This commit fixes both. + +commit 996cddd98b9e24f1bad900277fd12d85ce900d5d +Author: Armin Braun +Date: Wed Dec 4 08:41:42 2019 +0100 + + Stop Copying Every Http Request in Message Handler (#44564) (#49809) + + * Copying the request is not necessary here. We can simply release it once the response has been generated and a lot of `Unpooled` allocations that way + * Relates #32228 + * I think the issue that preventet that PR that PR from being merged was solved by #39634 that moved the bulk index marker search to ByteBuf bulk access so the composite buffer shouldn't require many additional bounds checks (I'd argue the bounds checks we add, we save when copying the composite buffer) + * I couldn't neccessarily reproduce much of a speedup from this change, but I could reproduce a very measureable reduction in GC time with e.g. Rally's PMC (4g heap node and bulk requests of size 5k saw a reduction in young GC time by ~10% for me) + +commit c33be29dc7a1768b1b1a7597e129de3671bfae52 +Author: Hendrik Muhs +Date: Wed Dec 4 07:55:57 2019 +0100 + + [Transform] automatic deletion of old checkpoints (#49496) + + add automatic deletion of old checkpoints based on count and time + +commit 602369c8b9aea9d353c3166e989e00f60313cfbe +Author: Mark Vieira +Date: Tue Dec 3 16:18:49 2019 -0800 + + Update CI BWC versions + +commit 0f27c0b70292b59e18039a48033d5672ae51282e +Author: Jason Tedor +Date: Tue Dec 3 14:20:32 2019 -0500 + + Extend systemd timeout during startup (#49784) + + When we are notifying systemd that we are fully started up, it can be + that we do not notify systemd before its default timeout of sixty + seconds elapses (e.g., if we are upgrading on-disk metadata). In this + case, we need to notify systemd to extend this timeout so that we are + not abruptly terminated. We do this by repeatedly sending + EXTEND_TIMEOUT_USEC to extend the timeout by thirty seconds; we do this + every fifteen seconds. This will prevent systemd from abruptly + terminating us during a long startup. We cancel the scheduled execution + of this notification after we have successfully started up. + +commit d5eb9379c9e8df80172ddcb3c40bc3ffe665e4d1 +Author: Hendrik Muhs +Date: Tue Dec 3 18:28:23 2019 +0100 + + remove flaky test: might fail due to async execution + +commit 7aae21228725d592a7a953f493921e89e16eef3a +Author: Hendrik Muhs +Date: Tue Dec 3 18:05:06 2019 +0100 + + [Transform] Fix possible audit logging disappearance after rolling upgrade (#49731) (#49767) + + ensure audit index template is available during a rolling upgrade before a + transform task can write to it. + + fixes #49730 + +commit a3f88595d7381a4d5e30257e708ed21619bd736f +Author: Przemysław Witek +Date: Tue Dec 3 15:48:39 2019 +0100 + + A few cleanups in evaluation tests (#49791) (#49794) + +commit fbb92f527ab6e5cbe841b55d8641774f0fa8826d +Author: Yannick Welsch +Date: Tue Dec 3 11:46:50 2019 +0100 + + Replicate write actions before fsyncing them (#49746) + + This commit fixes a number of issues with data replication: + + - Local and global checkpoints are not updated after the new operations have been fsynced, but + might capture a state before the fsync. The reason why this probably went undetected for so + long is that AsyncIOProcessor is synchronous if you index one item at a time, and hence working + as intended unless you have a high enough level of concurrent indexing. As we rely in other + places on the assumption that we have an up-to-date local checkpoint in case of synchronous + translog durability, there's a risk for the local and global checkpoints not to be up-to-date after + replication completes, and that this won't be corrected by the periodic global checkpoint sync. + - AsyncIOProcessor also has another "bad" side effect here: if you index one bulk at a time, the + bulk is always first fsynced on the primary before being sent to the replica. Further, if one thread + is tasked by AsyncIOProcessor to drain the processing queue and fsync, other threads can + easily pile more bulk requests on top of that thread. Things are not very fair here, and the thread + might continue doing a lot more fsyncs before returning (as the other threads pile more and + more on top), which blocks it from returning as a replication request (e.g. if this thread is on the + primary, it blocks the replication requests to the replicas from going out, and delaying + checkpoint advancement). + + This commit fixes all these issues, and also simplifies the code that coordinates all the after + write actions. + +commit 6e751f5536df79f287bb41d77b9f7dde7c2708f9 +Author: Alexander Reelsen +Date: Tue Dec 3 11:47:27 2019 +0100 + + Docs: Fix & test more grok processor documentation (#49447) + + The documentation contained a small error, as bytes and duration was not + properly converted to a number and thus remained a string. + + The documentation is now also properly tested by providing a full blown + simulate pipeline example. + +commit 0592b3c726425edb10b457e4d000466fd9a623b5 +Author: Colin Goodheart-Smithe +Date: Tue Dec 3 10:20:05 2019 +0000 + + Removes PR that was not in 7.5.0 release + +commit cdbcab384134a2d7d0c55ae50215052ed227e1e8 +Author: Christoph Büscher +Date: Tue Dec 3 10:24:55 2019 +0100 + + Remove Awaitsfix that seems outdated (#49764) + + This test seems to be fixed on 7.x with + https://github.com/elastic/elasticsearch/pull/43871 so the muting annotation can + most likely be removed. + +commit 1d8e3d69d74d4d0427e1157140339f43072a8dbc +Author: Przemysław Witek +Date: Tue Dec 3 09:54:16 2019 +0100 + + Make only a part of `stop()` method a critical section. (#49756) (#49788) + +commit 3bbaa017644e8a556444233bab2c5ec1221a1e9e +Author: Mayya Sharipova +Date: Mon Dec 2 17:01:30 2019 -0500 + + Disable sort optimization when index is sorted (#49727) + + Don't run long sort optimization when index is already + sorted on the same field as the sort query parameter. + + Relates to #37043, follow up for #48804 + +commit ad274dd7970ac882c685016ac29bdabb94fd3354 +Author: Mayya Sharipova +Date: Mon Dec 2 17:00:58 2019 -0500 + + Mute testIndexHasDuplicateData (#49779) + + Related to #49703 + +commit c4cc90be1cc3e9da3864e09f02901c2798720252 +Author: cachedout +Date: Mon Dec 2 21:31:47 2019 +0000 + + Recommend Metricbeat for 7.x (#49758) + + * Recommend Metricbeat for 7.x + + * Fix typo in link to configuring-metricbeat + + * [DOCS] Fixes build error and some terminology + + * Add to local exporter page per review feedback + +commit f1fd41cb53cc8db8b7675bb0d9622c7c1b8579e1 +Author: James Rodewig +Date: Mon Dec 2 15:52:13 2019 -0500 + + [DOCS] Document CCR compatibility requirements (#49776) + + * Creates a prerequisites section in the cross-cluster replication (CCR) + overview. + * Adds concise definitions for local and remote cluster in a CCR context. + * Documents that the ES version of the local cluster must be the same + or a newer compatible version as the remote cluster. + +commit ebc13ca498a7a61edcc1f7e1f583bb457819a8f4 +Author: Paul Sanwald +Date: Mon Dec 2 15:03:38 2019 -0500 + + re-categorize things that appeared in multiple area labels (#49777) + +commit a5dc6e062ec61f205da43b0ab35aa5dada5f326c +Author: Hendrik Muhs +Date: Mon Dec 2 20:53:05 2019 +0100 + + Document issue 49730 in release notes for 7.5.0 (#49733) + + document low severity issue about transform audit index potentially disappearing during rolling upgrade + + See #49730 for details + +commit 3eae180b8b2d2fd3b9e0ee2f3c8aec7a0527de91 +Author: jimczi +Date: Mon Dec 2 20:14:47 2019 +0100 + + add new version 7.5.1 + +commit 5f766a66fb9834443e4e4bc38fd026b4dfca024d +Author: Armin Braun +Date: Mon Dec 2 19:14:34 2019 +0100 + + Make Snapshot Metadata Javadocs Clearer (#49697) (#49771) + + We are always using the snapshot name on the shard level, + lets make it crystal clear in the docs. + +commit 96f14fcfbdbd39462cd5f77d626e4d2062b9912c +Author: lcawl +Date: Mon Dec 2 08:16:23 2019 -0800 + + [DOCS] Removes coming tags + +commit e6dc5bf9c2feb92edf7acccebfe2517ab522fd6b +Author: Jim Ferenczi +Date: Mon Dec 2 16:43:36 2019 +0100 + + Add release highlights for 7.5.0 (#49320) + +commit e2982b2110b047ae3ede46eb2621d7bfb68a70b3 +Author: Andrei Stefan +Date: Mon Dec 2 16:05:05 2019 +0200 + + SQL: handle NULL arithmetic operations with INTERVALs (#49633) + + (cherry picked from commit ce727615c08cf5ae422feb77f69ea24fb53cd9d1) + +commit 34311dd8181d1f546d0a28ccbc5f415fa39d80a0 +Author: Andrei Stefan +Date: Mon Dec 2 16:04:28 2019 +0200 + + Fix NULL handling for FLOOR and CEIL math functions (#49644) + + (cherry picked from commit 034f4cf7b4bd062c157d40f1e7a8760de31de568) + +commit 4dc83a7db9f5af8768e71e90efd7f6d37e71e280 +Author: Andrei Stefan +Date: Mon Dec 2 16:03:52 2019 +0200 + + Fix Locate function optional parameter handling (#49666) + + (cherry picked from commit dd3aeb8f5497bec4b050beaaf9d628a179b5454f) + +commit ff00174b614eddadb5a553e45a88caff4b78e302 +Author: Ignacio Vera +Date: Mon Dec 2 16:21:56 2019 +0100 + + Add CoreValuesSourceTypeTests for histogram (#49751) (#49765) + +commit ade72b97b76e731ca115c5079cd7dfa9d67c1cd0 +Author: James Rodewig +Date: Mon Dec 2 09:22:21 2019 -0500 + + [DOCS] Reformat keep types and keep words token filter docs (#49604) + + * Adds title abbreviations + * Updates the descriptions and adds Lucene links + * Reformats parameter definitions + * Adds analyze and custom analyzer snippets + * Adds explanations of token types to keep types token filter and tokenizer docs + +commit 86a40f6d8b078cf2eaafbe164872ed064dd50a29 +Author: David Turner +Date: Mon Dec 2 12:43:18 2019 +0000 + + Drop snapshot instructions for autobootstrap fix (#49755) + + The "Restore any snapshots as required" step is a trap: it's somewhere between + tricky and impossible to restore multiple clusters into a single one. + + Also add a note about configuring discovery during a rolling upgrade to + proscribe any rare cases where you might accidentally autobootstrap during the + upgrade. + +commit 3d44c1163ae39dd2d305476fbc2e7a1c9fd293b6 +Author: James Rodewig +Date: Mon Dec 2 09:12:21 2019 -0500 + + [DOCS] Explicitly document enrich `target_field` includes `match_field` (#49407) + + When the enrich processor appends enrich data to an incoming document, + it adds a `target_field` to contain the enrich data. + + This `target_field` contains both the `match_field` AND `enrich_fields` + specified in the enrich policy. + + Previously, this was reflected in the documented example but not + explicitly stated. This adds several explicit statements to the docs. + +commit 04ace7a6da6522a7b3a066c6c25d3700c2cf79f2 +Author: Christoph Büscher +Date: Mon Dec 2 11:21:34 2019 +0100 + + Add note how to run locale sensitive unit test (#49491) + + Some unit test checking locale sensitive functionality require the + -Djava.locale.providers=SPI,COMPAT flag to be set. When running tests though + gradle we pass this already to the BuildPlugin, but running from the IDE this + might need to be set manually. Adding a note explaining this to the + CONTRIBUTING.md doc and leaving a note in the test comment of + SearchQueryIT.testRangeQueryWithLocaleMapping which is a test we know + that suffers from this issue. + +commit d363b6b8ae4606a9a1787a1ddc978e68bbefeba4 +Author: Przemyslaw Gomulka +Date: Mon Dec 2 10:07:07 2019 +0100 + + Run 180_locale_dependent_mapping only since 6.8.5 (#49724) + + closes #49719 + +commit 5adb33ec177a24412b4cabbd561d9291029f18c9 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Sun Dec 1 19:24:27 2019 +0100 + + Deprecate sorting in reindex (#49458) (#49738) + + Reindex sort never gave a guarantee about the order of documents being + indexed into the destination, though it could give a sense of locality + of source data. + + It prevents us from doing resilient reindex and other optimizations and + it has therefore been deprecated. + + Related to #47567 + +commit 62a891bfa3369018b08fb6f85173507d860e1f43 +Author: Mayya Sharipova +Date: Fri Nov 29 16:51:50 2019 -0500 + + Add bulkScorer to script score query (#46336) (#49734) + + Some queries return bulk scorers that can be significantly faster than + iterating naively over the scorer. By giving script_score a BulkScorer + that would delegate to the wrapped query, we could make it faster in some cases. + + Closes #40837 + +commit 1d745f1e5c08e9869923c5e5eae7f89c3b0b537a +Author: Henning Andersen +Date: Fri Nov 29 22:07:55 2019 +0100 + + Revert "Deprecate sorting in reindex (#49458)" + + This reverts commit 27d45c9f1fff1016c3831d9f3cbf5750a0b54b60. + +commit 7cf170830c554257dc81006494d908dcc4b1b0ae +Author: Mayya Sharipova +Date: Fri Nov 29 15:37:40 2019 -0500 + + Optimize sort on numeric long and date fields. (#49732) + + This rewrites long sort as a `DistanceFeatureQuery`, which can + efficiently skip non-competitive blocks and segments of documents. + Depending on the dataset, the speedups can be 2 - 10 times. + + The optimization can be disabled with setting the system property + `es.search.rewrite_sort` to `false`. + + Optimization is skipped when an index has 50% or more data with + the same value. + + Optimization is done through: + 1. Rewriting sort as `DistanceFeatureQuery` which can + efficiently skip non-competitive blocks and segments of documents. + + 2. Sorting segments according to the primary numeric sort field(#44021) + This allows to skip non-competitive segments. + + 3. Using collector manager. + When we optimize sort, we sort segments by their min/max value. + As a collector expects to have segments in order, + we can not use a single collector for sorted segments. + We use collectorManager, where for every segment a dedicated collector + will be created. + + 4. Using Lucene's shared TopFieldCollector manager + This collector manager is able to exchange minimum competitive + score between collectors, which allows us to efficiently skip + the whole segments that don't contain competitive scores. + + 5. When index is force merged to a single segment, #48533 interleaving + old and new segments allows for this optimization as well, + as blocks with non-competitive docs can be skipped. + + Backport for #48804 + + + Co-authored-by: Jim Ferenczi + +commit 27d45c9f1fff1016c3831d9f3cbf5750a0b54b60 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Nov 29 17:46:44 2019 +0100 + + Deprecate sorting in reindex (#49458) + + Reindex sort never gave a guarantee about the order of documents being + indexed into the destination, though it could give a sense of locality + of source data. + + It prevents us from doing resilient reindex and other optimizations and + it has therefore been deprecated. + + Related to #47567 + +commit dcb9d5177cca20e4797527d6232d97f7d4148074 +Author: Tugberk Ugurlu +Date: Fri Nov 29 17:42:02 2019 +0000 + + [Docs] Fix typo in templates.asciidoc (#49726) + +commit 901a8d1dccbe5087e41523c3195bd8283d496260 +Author: Marios Trivyzas +Date: Fri Nov 29 17:06:26 2019 +0100 + + SQL: Fix issues with WEEK/ISO_WEEK/DATEDIFF (#49405) + + Some extended testing with MS-SQL server and H2 (which agree on + results) revealed bugs in the implementation of WEEK related extraction + and diff functions. + + Non-iso WEEK seems to be broken since #48209 because + of the replacement of Calendar and the change in the ISO rules. + + ISO_WEEK failed for some edge cases around the January 1st. + + DATE_DIFF was previously based on non-iso WEEK extraction which seems + not to be the case. + + Fixes: #49376 + + (cherry picked from commit 54fe7f57289c46bb0905b1418f51a00e8c581560) + +commit d9162c1243c092cb8f8477353382a4427e545291 +Author: Ignacio Vera +Date: Fri Nov 29 15:47:23 2019 +0100 + + Replace usages of XPackPlugin with the LocalStateCompositeXPackPlugin (#49714) (#49722) + +commit c2d316a22fa46e88c8766f5e3bf18f11685c7204 +Author: Yannick Welsch +Date: Fri Nov 29 11:01:24 2019 +0100 + + Remove obsolete resolving logic from TRA (#49685) + + This stems from a time where index requests were directly forwarded to + TransportReplicationAction. Nowadays they are wrapped in a BulkShardRequest, and this logic is + obsolete. + + In contrast to prior PR (#49647), this PR also fixes (see b3697cc) a situation where the previous + index expression logic had an interesting side effect. For bulk requests (which had resolveIndex + = false), the reroute phase was waiting for the index to appear in case where it was not present, + and for all other replication requests (resolveIndex = true) it would right away throw an + IndexNotFoundException while resolving the name and exit. With #49647, every replication + request was now waiting for the index to appear, which was problematic when the given index + had just been deleted (e.g. deleting a follower index while it's still receiving requests from the + leader, where these requests would now wait up to a minute for the index to appear). This PR + now adds b3697cc on top of that prior PR to make sure to reestablish some of the prior behavior + where the reroute phase waits for the bulk request for the index to appear. That logic was in + place to ensure that when an index was created and not all nodes had learned about it yet, that + the bulk would not fail somewhere in the reroute phase. This is now only restricted to the + situation where the current node has an older cluster state than the one that coordinated the + bulk request (which checks that the index is present). This also means that when an index is + deleted, we will no longer unnecessarily wait up to the timeout for the index o appear, and + instead fail the request. + + Closes #20279 + +commit 4edb2e7bb62dd0b58b3eb15b4101b4186baac8cb +Author: Dimitris Athanasiou +Date: Fri Nov 29 16:10:44 2019 +0200 + + [7.x][ML] Add optional source filtering during data frame reindexing (#49690) (#49718) + + This adds a `_source` setting under the `source` setting of a data + frame analytics config. The new `_source` is reusing the structure + of a `FetchSourceContext` like `analyzed_fields` does. Specifying + includes and excludes for source allows selecting which fields + will get reindexed and will be available in the destination index. + + Closes #49531 + + Backport of #49690 + +commit 813b49adb42d935d1b1bcd86bd54403e7ebdbf0d +Author: Armin Braun +Date: Fri Nov 29 14:57:47 2019 +0100 + + Make BlobStoreRepository Aware of ClusterState (#49639) (#49711) + + * Make BlobStoreRepository Aware of ClusterState (#49639) + + This is a preliminary to #49060. + + It does not introduce any substantial behavior change to how the blob store repository + operates. What it does is to add all the infrastructure changes around passing the cluster service to the blob store, associated test changes and a best effort approach to tracking the latest repository generation on all nodes from cluster state updates. This brings a slight improvement to the consistency + by which non-master nodes (or master directly after a failover) will be able to determine the latest repository generation. It does not however do any tricky checks for the situation after a repository operation + (create, delete or cleanup) that could theoretically be used to get even greater accuracy to keep this change simple. + This change does not in any way alter the behavior of the blobstore repository other than adding a better "guess" for the value of the latest repo generation and is mainly intended to isolate the actual logical change to how the + repository operates in #49060 + +commit 3d525e18f983355a79f8e51016768056ef290a04 +Author: Dimitrios Liappis +Date: Fri Nov 29 15:53:41 2019 +0200 + + Mute MixedClusterClientYamlTestSuiteIT (#49721) + + Details in https://github.com/elastic/elasticsearch/issues/49719 + Relates https://github.com/elastic/elasticsearch/pull/47983 + +commit 90e9d61f2b75aa2962685175ea1bd92b8bb7c223 +Author: Armin Braun +Date: Fri Nov 29 11:17:47 2019 +0100 + + Optimize GoogleCloudStorageHttpHandler (#49677) (#49707) + + Removing a lot of needless buffering and array creation + to reduce the significant memory usage of tests using this. + The incoming stream from the `exchange` is already buffered + so there is no point in adding a ton of additional buffers + everywhere. + +commit a59b7e07f15e1509e78f887f1b6e0b76f9a6953c +Author: Ioannis Kakavas +Date: Fri Nov 29 09:43:55 2019 +0200 + + Use PEM files instead of a JKS for key material (#49625) (#49701) + + So that the tests can also run in a FIPS 140 JVM, where using a + JKS keystore is not allowed. + + Resolves: #49261 + +commit e6f530c1671dba09ac487f63fa4ef2f836093fe2 +Author: Tim Vernum +Date: Fri Nov 29 15:01:20 2019 +1100 + + Improved diagnostics for TLS trust failures (#49669) + + - Improves HTTP client hostname verification failure messages + - Adds "DiagnosticTrustManager" which logs certificate information + when trust cannot be established (hostname failure, CA path failure, + etc) + + These diagnostic messages are designed so that many common TLS + problems can be diagnosed based solely (or primarily) on the + elasticsearch logs. + + These diagnostics can be disabled by setting + + xpack.security.ssl.diagnose.trust: false + + Backport of: #48911 + +commit 31f13e839c33a0adba56130c3cf91c87e39132f3 +Author: Tim Vernum +Date: Fri Nov 29 12:59:16 2019 +1100 + + Correct the documentation for create_doc privilege (#49354) + + The documentation was added in #47584 but those docs did not reflect the up-to-date behavior of the feature. + + Backport of: #47784 + +commit ba0c84802768bb37a9fb623377de3143a758a0c5 +Author: Ioannis Kakavas +Date: Fri Nov 29 00:17:16 2019 +0200 + + [7.x] Update opensaml dependency (#44972) (#49512) + + Add a mirror of the maven repository of the shibboleth project + and upgrade opensaml and related dependencies to the latest + version available version + + Resolves: #44947 + +commit 1425e30b1e0d8dbfce3c0286326e23421cd0f05e +Author: Przemysław Witek +Date: Thu Nov 28 21:46:46 2019 +0100 + + [7.x] Remove ClassInfo interface and BinaryClassInfo class. (#49649) (#49681) + +commit 496bb9e2ee878318703289aed246df359a994503 +Author: Jim Ferenczi +Date: Thu Nov 28 18:23:09 2019 +0100 + + Add a listener to track the progress of a search request locally (#49471) (#49691) + + This commit adds a function in NodeClient that allows to track the progress + of a search request locally. Progress is tracked through a SearchProgressListener + that exposes query and fetch responses as well as partial and final reduces. + This new method can be used by modules/plugins inside a node in order to track the + progress of a local search request. + + Relates #49091 + +commit 2dafecc39894ef48407c70dea1a4381dda6a0a54 +Author: Mayya Sharipova +Date: Thu Nov 28 11:59:58 2019 -0500 + + Upgrade lucene to 8.4.0-snapshot-e648d601efb (#49641) + +commit d5842aebab5d42a64b2e05fba94bf4cf2b7f3cbc +Author: Marios Trivyzas +Date: Thu Nov 28 17:00:41 2019 +0100 + + [Docs] Enhance rolling upgrade guide (#49686) + + Add a couple of pointers for the user to check the + overall cluster health and the version of ES running + on every node. + + Fixes: #49670 + + (cherry picked from commit 8ca11f54cd839f41632c556601e94da67e91a3d1) + +commit 1824a2fa588fab8ed650b37df795776ec9ab101e +Author: Adrien Grand +Date: Thu Nov 28 15:54:32 2019 +0100 + + Pure disjunctions should rewrite to a MatchNoneQueryBuilder (#48557) (#49673) + + Closes #48475 + +commit e528b41cf24904dd795f5a5d059eb7b3804abaea +Author: Przemyslaw Gomulka +Date: Thu Nov 28 15:26:54 2019 +0100 + + Enable LicenceServiceTests for all jdks (#49440) backport(#49682) + + This test no longer relies on jdk version, so the assume should be removed + relates #48209 + +commit 326fe7566eff6ac78ae519357f9aeb44e6ada2df +Author: Ignacio Vera +Date: Thu Nov 28 15:06:26 2019 +0100 + + New Histogram field mapper that supports percentiles aggregations. (#48580) (#49683) + + This commit adds a new histogram field mapper that consists in a pre-aggregated format of numerical data to be used in percentiles aggregations. + +commit 04e9cbd6eb7677541a6bd9eb65fc3ce2ac5c7ca9 +Author: Yannick Welsch +Date: Thu Nov 28 13:12:07 2019 +0100 + + Revert "Remove obsolete resolving logic from TRA (#49647)" + + This reverts commit 0827ea2175ae0f8d05c17ba010d15a9ff616c806. + +commit 0827ea2175ae0f8d05c17ba010d15a9ff616c806 +Author: Yannick Welsch +Date: Thu Nov 28 11:31:51 2019 +0100 + + Remove obsolete resolving logic from TRA (#49647) + + This stems from a time where index requests were directly forwarded to + TransportReplicationAction. Nowadays they are wrapped in a BulkShardRequest, and this logic is + obsolete. + + Closes #20279 + +commit 35732504ba71382974dfce5bfeb858fa0352e866 +Author: jimczi +Date: Thu Nov 28 11:08:15 2019 +0100 + + #49166 Fix spurious test failure + +commit d6445fae4bb71841ed1f19a955f91c048f9c27b0 +Author: Jim Ferenczi +Date: Wed Nov 27 13:38:09 2019 +0100 + + Add a cluster setting to disallow loading fielddata on _id field (#49166) + + This change adds a dynamic cluster setting named `indices.id_field_data.enabled`. + When set to `false` any attempt to load the fielddata for the `_id` field will fail + with an exception. The default value in this change is set to `false` in order to prevent + fielddata usage on this field for future versions but it will be set to `true` when backporting + to 7x. When the setting is set to true (manually or by default in 7x) the loading will also issue + a deprecation warning since we want to disallow fielddata entirely when https://github.com/elastic/elasticsearch/issues/26472 + is implemented. + + Closes #43599 + +commit b236076f88b86b4f88a8fd4b6b5d20b8a6711686 +Author: Ryan Ernst +Date: Wed Nov 27 16:16:56 2019 -0800 + + Fix java 8 compile for Files.readString usage + +commit 7802b60a5a790629d36b826ef04bf01157746275 +Author: Ryan Ernst +Date: Wed Nov 27 16:01:26 2019 -0800 + + Migrate setup passwords packaging test from bats (#49337) + + This commit moves the packaging tests for elasticsearch-setup-passwords + to java from bats. The change also enables future tests to enable + security in Elasticsearch and automatically have waitForElasticsearch + work correctly, at least to the same extent it worked in bats, by + waiting on the ES port instead of health check. + + relates #46005 + +commit 9c6a5a09af1c8c6e2d04e8ec9563c05ac2409811 +Author: Ryan Ernst +Date: Wed Nov 27 11:19:34 2019 -0800 + + Make docker build task incremental (#49613) + + This commits sets an output marker file for the docker build tasks so + that it can be tracked as up to date. It also fixes the docker build + context task to omit the build date as in input property which always + left the task as out of date. + + relates #49359 + +commit d66795fdf02a1ef0b72db5265bfbb4ee3f8aa68a +Author: Christos Soulios <1561376+csoulios@users.noreply.github.com> +Date: Wed Nov 27 20:50:27 2019 +0200 + + Fix typo when assigning null_value in GeoPointFieldMapper (#49655) + + Backport of #49645 to 7.x + This PR fixes a trivial typo error that affects assigning null_value in the GeoPointFieldMapper + +commit f2886960409a298fbf67514de3c89d7f936c13f8 +Author: Ryan Ernst +Date: Wed Nov 27 10:42:15 2019 -0800 + + Remove legacy referene to file scripts (#49339) + + This commit removes outdated documentation about a path setting for file + scripts which no longer exist. + + closes #45827 + +commit 297efa83247ef6c061e164096dd63342ed56a7d6 +Author: Ryan Ernst +Date: Wed Nov 27 10:39:54 2019 -0800 + + Add JAVA_HOME env override location to docs (#49565) + + This commit clarifies how to override JAVA_HOME from the bundled jdk for + deb and rpm installs, which each have their own file that is sourced + upon service startup. + + closes #49068 + +commit 0a42395dfa5a999eb10eb553dc5f467426000c5d +Author: Martijn van Groningen +Date: Wed Nov 27 15:53:40 2019 +0100 + + Backport: add templating support to pipeline processor (#49643) + + Backport of #49030 + + This commit adds templating support to the pipeline processor's `name` option. + + Closes #39955 + +commit 15342c4dd2ed24f7e7b9d6ecdb4f44f650e9af68 +Author: Xiang Dai <764524258@qq.com> +Date: Wed Nov 27 22:39:34 2019 +0800 + + [DOCS] Clarify how to update max memory size in bootstrap checks (#48975) + +commit 2e0b079a16e55e17958781ba371a543831176fc3 +Author: bellengao +Date: Wed Nov 27 22:26:57 2019 +0800 + + [DOCS] Correct the request path for flush API docs (#49615) + +commit 5ffb615b379e76610469702c5006e4da59e2244c +Author: Yannick Welsch +Date: Wed Nov 27 14:43:21 2019 +0100 + + [DOCS] Correct request path for synced flush API docs (#49631) + + Fixes an incorrect request path added with #46634 + +commit d1ed2ae25b7ba80d695f4911216defd29a8ad486 +Author: glerb <56281590+glerb@users.noreply.github.com> +Date: Wed Nov 27 05:36:59 2019 -0800 + + [Docs] Correct typo in log file name (#49620) + +commit 4b6915ea41ede625422e48310038513abd882511 +Author: Dimitrios Liappis +Date: Wed Nov 27 13:42:54 2019 +0200 + + Clarify gid used by docker image process and bind-mount method (#49632) + + Fix reference about the uid:gid that Elasticsearch runs as inside + the Docker container and add a packaging test to ensure that bind + mounting a data dir with a random uid and gid:0 works as + expected. + + Backport of #49529 + Closes #47929 + +commit 502873b1447a44d1ef34d377eb0dd73c54ee266d +Author: Przemyslaw Gomulka +Date: Wed Nov 27 12:29:18 2019 +0100 + + [Java.time] Retain prefixed date pattern in formatter (#48703) + + JavaDateFormatter should keep the pattern with the prefixed 8 as it will be used for serialisation. The stripped pattern should be used for the enclosed formatters. + + closes #48698 + +commit 0a73ba05de30b67b66e3bea84ac6e59d2730de1d +Author: Yannick Welsch +Date: Wed Nov 27 09:23:42 2019 +0100 + + Do not mutate request on scripted upsert (#49578) + + Fixes a bug where a scripted upsert that causes a dynamic mapping update is retried (because + mapping update is still in-flight), and the request is mutated multiple times. + + Closes #48670 + +commit bd007271cfc66b1663ff51f59d8c769b5120c85d +Author: Yannick Welsch +Date: Wed Nov 27 09:24:16 2019 +0100 + + Avoid double-wrapping allocator (#49534) + + When using unpooled, the allocator is wrapped twice in a NoDirectBuffers. + +commit 09c42690971682f3045988b554f8f14d3ebd9e91 +Author: Martijn van Groningen +Date: Wed Nov 27 07:52:42 2019 +0100 + + Add templating support to enrich processor (#49093) + + Adds support for templating to `field` and `target_field` options. + +commit 90850f4ea01824e10a6cd51ba8695b05e6249dff +Author: Martijn van Groningen +Date: Wed Nov 27 07:52:08 2019 +0100 + + Backport: Introduce on_failure_pipeline ingest metadata inside on_failure block (#49596) + + Backport of #49076 + + In case an exception occurs inside a pipeline processor, + the pipeline stack is kept around as header in the exception. + Then in the on_failure processor the id of the pipeline the + exception occurred is made accessible via the `on_failure_pipeline` + ingest metadata. + + Closes #44920 + +commit 901c64ebbf9f8e8d57574dbd3ac22a9ef1622cc8 +Author: Tim Vernum +Date: Wed Nov 27 16:39:07 2019 +1100 + + Add Debug/Trace logging for authentication (#49619) + + Authentication has grown more complex with the addition of new realm + types and authentication methods. When user authentication does not + behave as expected it can be difficult to determine where and why it + failed. + + This commit adds DEBUG and TRACE logging at key points in the + authentication flow so that it is possible to gain addition insight + into the operation of the system. + + Backport of: #49575 + +commit e9ad1a7fcd83337df0e7b71208a66ee4527a3866 +Author: Tim Vernum +Date: Wed Nov 27 13:46:52 2019 +1100 + + Fix iterate-from-1 bug in smart realm order (#49614) + + The AuthenticationService has a feature to "smart order" the realm + chain so that whicherver realm was the last one to successfully + authenticate a given user will be tried first when that user tries to + authenticate again. + + There was a bug where the building of this realm order would + incorrectly drop the first realm from the default chain unless that + realm was the "last successful" realm. + + In most cases this didn't cause problems because the first realm is + the reserved realm and so it is unusual for a user that authenticated + against a different realm to later need to authenticate against the + resevered realm. + + This commit fixes that bug and adds relevant asserts and tests. + + Backport of: #49473 + +commit e965a6f2dfd7ec375c940337615d559069e53d18 +Author: Tim Brooks +Date: Tue Nov 26 16:37:27 2019 -0700 + + Fix remote settings upgrade test (#49609) + + This commit fixes #49587. Due to a settings change, the broken test was + asserting on the incorrect setting. This commit fixes that issue and + adds additional assertions to ensure that all settings are working + properly. + +commit 996cdebfb426e9ad6190f62c82fd99f9ecaced70 +Author: Armin Braun +Date: Tue Nov 26 22:37:31 2019 +0100 + + Make BlobStoreRepository#writeIndexGen API Async (#49584) (#49610) + + Preliminary to shorten the diff of #49060. In #49060 + we execute cluster state updates during the writing of a new + index gen and thus it must be an async API. + +commit 386240027086ffa8b8fecacb6856e79635a8154c +Author: Armin Braun +Date: Tue Nov 26 20:57:19 2019 +0100 + + Remove Redundant EsBlobStoreTestCase (#49603) (#49605) + + All the implementations of `EsBlobStoreTestCase` use the exact same + bootstrap code that is also used by their implementation of + `EsBlobStoreContainerTestCase`. + This means all tests might as well live under `EsBlobStoreContainerTestCase` + saving a lot of code duplication. Also, there was no HDFS implementation for + `EsBlobStoreTestCase` which is now automatically resolved by moving the tests over + since there is a HDFS implementation for the container tests. + +commit 777431265b713b01696abe87a4136dcfa98126ec +Author: lcawl +Date: Tue Nov 26 10:28:18 2019 -0800 + + [DOCS] Fixes typo in ML resources + +commit fe2c65185ea0c89f3a4bbc1f662fccf6c726aa81 +Author: Alan Woodward +Date: Tue Nov 26 16:51:41 2019 +0000 + + Annotated text type should extend TextFieldType (#49555) + + The annotated text mapper has a field type that currently extends StringFieldType, + which means that all the positional-related query factory methods need to be copied + over from TextFieldType. In addition, MappedFieldType.intervals() hasn't been + overridden, so you can't use intervals queries with annotated text - a major drawback, + since one of the purposes of annotated text is to be able to run positional queries against + annotations. + + This commit changes the annotated text field type to extend TextFieldType instead, + adding tests to ensure that position queries work correctly. + + Closes #49289 + +commit b5d7c939f86ad5c0493cb51d9c5034772750e3e5 +Author: Benjamin Trent +Date: Tue Nov 26 11:28:26 2019 -0500 + + [7.x] [ML][Inference][HLRC] add GET _stats (#49562) (#49600) + + * [ML][Inference][HLRC] add GET _stats (#49562) + + * fixing for backport + +commit a42003b95b4c196ec4269a29fe783ce0ee7aa199 +Author: lcawl +Date: Tue Nov 26 08:21:39 2019 -0800 + + [DOCS] Fixes data type formatting + +commit 495b543e63da7b8c39876f737daa59d8300ca8c1 +Author: Armin Braun +Date: Tue Nov 26 16:53:51 2019 +0100 + + Improve Stability of GCS Mock API (#49592) (#49597) + + Same as #49518 pretty much but for GCS. + Fixing a few more spots where input stream can get closed + without being fully drained and adding assertions to make sure + it's always drained. + Moved the no-close stream wrapper to production code utilities since + there's a number of spots in production code where it's also useful + (will reuse it there in a follow-up). + +commit 26a8ca00dbc5237887c7da3695739a57f8c8fc2d +Author: Benjamin Trent +Date: Tue Nov 26 08:27:08 2019 -0500 + + [7.x] [ML][Inference][HLRC] Delete trained model API (#49567) (#49585) + + * [ML][Inference][HLRC] Delete trained model API (#49567) + + * fixing for backport + +commit b0cb7bf229e3d386d08013c910888c67fe3c2ee4 +Author: Marios Trivyzas +Date: Tue Nov 26 13:23:41 2019 +0100 + + SQL: Fix issue with GROUP BY YEAR() (#49559) + + Grouping By YEAR() is translated to a histogram aggregation, but + previously if there was a scalar function invloved (e.g.: + `YEAR(date + INTERVAL 2 YEARS)`), there was no proper script created + and the histogram was applied on a field with name: `date + INTERVAL 2 YEARS` + which doesn't make sense, and resulted in null result. + + Check the underlying field of YEAR() and if it's a function call + `asScript()` to properly get the painless script on which the histogram + is applied. + + Fixes: #49386 + (cherry picked from commit 93c37abc943d00d3a14ba08435d118a6d48874c7) + +commit cf5f01303334499deb0bda61c714302a506df5d3 +Author: Rory Hunter +Date: Tue Nov 26 12:36:56 2019 +0000 + + Return 400 when handling invalid JSON (#49558) + + Backport of #49552. + + Closes #49428. The code that works out an HTTP code for an exception didn't + consider the JsonParseException case, meant that an invalid JSON request could + result in a 500 Internal Server Error. Now it returns 400 Bad Request. + +commit 41daf284f5b2e6faeb6b6d58a5095a21ca3aeba2 +Author: Hendrik Muhs +Date: Tue Nov 26 13:28:19 2019 +0100 + + mute FullClusterRestartSettingsUpgradeIT + +commit 3c69d4d0bd491954174fb21f664ff42aed8e4bcb +Author: Marios Trivyzas +Date: Tue Nov 26 12:30:49 2019 +0100 + + SQL: Add TRUNC alias for TRUNCATE (#49571) + + Add TRUNC as alias to already implemented TRUNCATE + numeric function which is the flavour supported by + Oracle and PostgreSQL. + + Relates to: #41195 + + (cherry picked from commit f2aa7f0779bc5cce40cc0c1f5e5cf1a5bb7d84f0) + +commit 048b9dbb145023095a24f116ccd1ec51a07267f4 +Author: j-bean +Date: Tue Nov 26 13:39:31 2019 +0300 + + Fix expired job results deletion audit message (#49560) + + The PR fixes #49549 + +commit c23a2187da5302ed2c0eb4753e5f81b8ca8c0267 +Author: Dimitris Athanasiou +Date: Tue Nov 26 12:20:37 2019 +0200 + + [7.x][ML] Only report complete writing_results progress after completion (#49551) (#49577) + + We depend on the number of data frame rows in order to report progress + for the writing of results, the last phase of a job run. However, results + include other objects than just the data frame rows (e.g, progress, inference model, etc.). + + The problem this commit fixes is that if we receive the last data frame row results + we'll report that progress is complete even though we still have more results to process + potentially. If the job gets stopped for any reason at this point, we will not be able + to restart the job properly as we'll think that the job was completed. + + This commit addresses this by limiting the max progress we can report for the + writing_results phase before the results processor completes to 98. + At the end, when the process is done we set the progress to 100. + + The commit also improves failure capturing and reporting in the results processor. + + Backport of #49551 + +commit 5d306ae3b29a23bf680d9154d250aee3f7175e08 +Author: Marios Trivyzas +Date: Tue Nov 26 10:47:14 2019 +0100 + + SQL: Fix issue with CASE/IIF pre-calculating results (#49553) + + Previously, CaseProcessor was pre-calculating (called `process()`) + on all the building elements of a CASE/IIF expression, not only the + conditions involved but also the results, as well as the final else result. + In case one of those results had an erroneous calculation + (e.g.: division by zero) this was executed and resulted in + an Exception to be thrown, even if this result was not used because of + the condition guarding it. e.g.: + + ``` + SELECT CASE myField1 = 0 THEN NULL ELSE myField2 / myField1 END + FROM test; + ``` + + Fixes: #49388 + (cherry picked from commit dbd169afc98686cae1bc72024fad0ca32b272efd) + +commit a4208e44f79817b0afbe882fab81a9acb207fef3 +Author: Christoph Büscher +Date: Tue Nov 26 10:46:44 2019 +0100 + + [Docs] Correct `max_doc_freq` default value (#49536) + + The default is set to Integer.MAX_VALUE but is reported to be `0` in the docs. + With the current implementation a value of 0 would mean all terms are filtered + out, which is the opposite of "unbounded". + + Closes #49520 + +commit 9cb1ace1c2d7ea49cb17470a0a37f3bfeed45f6c +Author: Tim Vernum +Date: Tue Nov 26 16:34:55 2019 +1100 + + Expand docs on TLSv1 breaking change (#49352) + + The breaking changes cover the removal of TLSv1 from the default + protocols, but assume that users who need to retain TLSv1 support will + understand all the places where they may used it. + + This has proven not to be true, as it is easy to be unaware that (for + example) an LDAP server is using TLSv1. + + This change explicitly lists all the places where TLS protocols may + need to be configured. + + Co-Authored-By: Lisa Cawley + Co-Authored-By: Pius + +commit d2e92a1791c9fe71c30bc3874f0b65231ade758e +Author: Nhat Nguyen +Date: Mon Nov 25 20:10:52 2019 -0500 + + EngineTestCase#getDocIds should use internal reader (#49564) + + We do not guarantee that EngineTestCase#getDocIds is called after the + engine has been externally refreshed. Hence, we trip an assertion + assertSearcherIsWarmedUp. + + CI: https://gradle-enterprise.elastic.co/s/pm2at5qmfm2iu + + Relates #48605 + +commit 416178c7c8644ea9ebee45a33fc680d6b642ab99 +Author: Tim Brooks +Date: Mon Nov 25 16:53:07 2019 -0700 + + Enable simple remote connection strategy (#49561) + + This commit back ports three commits related to enabling the simple + connection strategy. + + Allow simple connection strategy to be configured (#49066) + + Currently the simple connection strategy only exists in the code. It + cannot be configured. This commit moves in the direction of allowing it + to be configured. It introduces settings for the addresses and socket + count. Additionally it introduces new settings for the sniff strategy + so that the more generic number of connections and seed node settings + can be deprecated. + + The simple settings are not yet registered as the registration is + dependent on follow-up work to validate the settings. + + Ensure at least 1 seed configured in remote test (#49389) + + This fixes #49384. Currently when we select a random subset of seed + nodes from a list, it is possible for 0 seeds to be selected. This test + depends on at least 1 seed being selected. + + Add the simple strategy to cluster settings (#49414) + + This is related to #49067. This commit adds the simple connection + strategy settings and strategy mode setting to the cluster settings + registry. With these changes, the simple connection mode can be used. + Additionally, it adds validation to ensure that settings cannot be + misconfigured. + +commit 99e313695f449377a3a30430dc3c18ab0e07572d +Author: Zachary Tong +Date: Mon Nov 25 16:45:51 2019 -0500 + + Reuse CompensatedSum object in agg collect loops (#49548) + + The new CompensatedSum is a nice DRY refactor, but had the unanticipated + side effect of creating a lot of object allocation in the aggregation hot collection + loop: one object per visited document, per aggregator. In some places it + created two per-doc-per-agg (weighted avg, geo centroids, etc) since there + were multiple compensations being maintained. + + This PR moves the object creation out of the hot loop so that it is now + created once per segment, and resets the internal state each time through + the loop + +commit 2fd58bb84572e6dc502008a346172cba895c0135 +Author: James Rodewig +Date: Mon Nov 25 16:15:33 2019 -0500 + + [DOCS] Add missing "_type" to delimited payload token filter docs + +commit 26beb486c79488adade9c61ce41dfa8b7ed8166c +Author: Lisa Cawley +Date: Mon Nov 25 12:59:59 2019 -0800 + + [DOCS] Fixes security links (#49563) + +commit c40449ac22887ab0ab39d449a4a64fd9bb2e7d94 +Author: James Rodewig +Date: Mon Nov 25 15:38:52 2019 -0500 + + [DOCS] Reformat delimited payload token filter docs (#49380) + + * Adds a title abbreviation + * Relocates the older name deprecation warning + * Updates the description and adds a Lucene link + * Adds a note to explain payloads and how to store them + * Adds analyze and custom analyzer snippets + * Adds a 'Return stored payloads' example + +commit 99476db2d0b905c9fe4c7196e963722d4272966b +Author: James Rodewig +Date: Mon Nov 25 10:31:13 2019 -0500 + + [DOCS] Remove individual task retrieval from cat/tasks API (#49550) + +commit 688c78c58919bac3072a271ce6b95cd33cf93ef9 +Author: Benjamin Trent +Date: Mon Nov 25 10:09:30 2019 -0500 + + [ML] Stop timing stats failure propagation (#49495) (#49501) + +commit df5afa797ee1b6ec943a59b826dc882804cc41ba +Author: Kelly Campbell +Date: Mon Nov 25 09:36:38 2019 -0500 + + [DOCS] Correct GET path in cat tasks API docs (#49494) + + Previously, the request example included `GET _cat/_tasks`. However, the resource should be `tasks`, not `_tasks`. + +commit 62811c2272291eac44862ee71b513290b9eefad0 +Author: David Roberts +Date: Mon Nov 25 13:20:12 2019 +0000 + + [ML] Add default categorization analyzer definition to ML info (#49545) + + The categorization job wizard in the ML UI will use this + information when showing the effect of the chosen categorization + analyzer on a sample of input. + +commit d21df9eba96b8b73099162b8ed063ddf60f88fc0 +Author: Dimitris Athanasiou +Date: Mon Nov 25 15:12:41 2019 +0200 + + [ML][DOCS] Anomaly detection job retention days settings do not require restart (#49546) + +commit aca38f68826fc5210ac78dd599cfd3de1810b20c +Author: Dimitris Athanasiou +Date: Mon Nov 25 15:13:00 2019 +0200 + + [7.x][ML] DFA jobs should accept excluding an unsupported field (#49535) (#49544) + + Before this change excluding an unsupported field resulted in + an error message that explained the excluded field could not be + detected as if it doesn't exist. This error message is confusing. + + This commit commit changes this so that there is no error in this + scenario. When excluding a field that does exist but has been + automatically been excluded from the analysis there is no harm + (unlike excluding a missing field which could be a typo). + + Backport of #49535 + +commit 8c374014ae64469dd5230413d890ad32d58af169 +Author: Daniel Mitterdorfer +Date: Mon Nov 25 07:39:42 2019 -0500 + + Add note about Gradle wrapper on Windows (#49528) + + With this commit we add a clarifying note in the contribution guidelines + that our examples show the usage on Unix and also explain how to invoke + the Gradle wrapper script on Windows. + + Closes #49521 + +commit af0f97d50ab6f128f1fbe89efab6d6be5296b6ce +Author: Armin Braun +Date: Mon Nov 25 13:31:45 2019 +0100 + + Fix SLMSnapshotBlockingIntegTests.testSnapshotInProgress (#49533) (#49542) + + This test must check for state `SUCCESS` as well. `SUCESS` in + `SnapshotsInProgress` means "all data nodes finished snapshotting sucessfully but master must still finalize the snapshot in the repo". + `SUCESS` does not mean that the snapshot is actually fully finished in this object. + + You can easily reporduce the scenario in #49303 that has an in-progress snapshot in `SUCCESS` state + by waiting 20s before running the busy assert loop on the snapshot status so that all steps but the blocked + finalization can finish. + + Closes #49303 + +commit 2502ff39a03bca7327987b2efdfa90cf9733c278 +Author: Armin Braun +Date: Mon Nov 25 13:31:28 2019 +0100 + + Enhance SnapshotResiliencyTests (#49514) (#49541) + + A few enhancements to `SnapshotResiliencyTests`: + 1. Test running requests from random nodes in more spots to enhance coverage (this is particularly motivated by #49060 where the additional number of cluster state updates makes it more interesting to fully cover all kinds of network failures) + 2. Fix issue with restarting only master node in one test (doing so breaks the test at an incredibly low frequency, that becomes not so low in #49060 with the additional cluster state updates between request and response) + 3. Improved cluster formation checks (now properly checks the term as well when forming cluster) + makes sure all nodes are connected to all other nodes (previously the data nodes would at times not be connected to other data nodes, which was shaken out now by adding the `client()` method + 4. Make sure the cluster left behind by the test makes sense by running the repo cleanup action on it (this also increases coverage of the repository cleanup action obviously and adds the basis of making it part of more resiliency tests) + +commit c149c64dc454a1e2e71771559a3956b4fca640cd +Author: Dimitris Athanasiou +Date: Mon Nov 25 12:51:57 2019 +0200 + + [7.x][ML] Apply source query on data frame analytics memory estimation (#49517) (#49532) + + Closes #49454 + + Backport of #49517 + +commit a5fa86ed97cc497c6f694ed1dc19699640ebca24 +Author: Armin Braun +Date: Mon Nov 25 10:28:55 2019 +0100 + + Improve Stability of Mock APIs (#49518) (#49524) + + This commit ensures that even for requests that are known to be empty body + we at least attempt to read one bytes from the request body input stream. + This is done to work around the behavior in `sun.net.httpserver.ServerImpl.Dispatcher#handleEvent` + that will close a TCP/HTTP connection that does not have the `eof` flag (see `sun.net.httpserver.LeftOverInputStream#isEOF`) + set on its input stream. As far as I can tell the only way to set this flag is to do a read when there's no more bytes buffered. + This fixes the numerous connection closing issues because the `ServerImpl` stops closing connections that it thinks + weren't fully drained. + + Also, I removed a now redundant drain loop in the Azure handler as well as removed the connection closing in the error handler's + drain action (this shouldn't have an effect but makes things more predictable/easier to reason about IMO). + + I would suggest merging this and closing related issue after verifying that this fixes things on CI. + + The way to locally reproduce the issues we're seeing in tests is to make the retry timings more aggressive in e.g. the azure tests + and move them to single digit values. This makes the retries happen quickly enough that they run into the async connecting closing + of allegedly non-eof connections by `ServerImpl` and produces the exact kinds of failures we're seeing currently. + + Relates #49401, #49429 + +commit 5256756879e8e757c05546fec19a2d3bea983082 +Author: Hendrik Muhs +Date: Mon Nov 25 09:42:43 2019 +0100 + + [Transform] add debug log for configuration index (#49484) + + add debug log for transform creation and disallow partial results for retrieval + +commit 8260cba629690633fb754555030d8fe017f71a17 +Author: Nhat Nguyen +Date: Fri Nov 22 08:51:00 2019 -0500 + + Increase timeout while checking for no snapshotted commit (#49461) + + If some replica is performing a file-based recovery, then the check + assertNoSnapshottedIndexCommit would fail. We should increase the + timeout for this check so that we can wait until all recoveries done + or aborted. + + Closes #49403 + +commit 1d2bfd1af68a15b7e2b29c6d5983f88cc40efdb1 +Author: Jared Tan +Date: Mon Nov 25 02:07:11 2019 +0800 + + Include id to the error msg when it's too long (#49433) + +commit 777f6d5da606e2735a0e74ba5f9cb2e1121d4e9c +Author: Mark Vieira +Date: Fri Nov 22 17:19:47 2019 -0800 + + Fix extraction of notarized Elasticsearch release distribution (#49511) + + This commit introduces a workaround for an issue related to our recent + notarization of distributions starting with the 6.8.5 release. An + unintended side effect of notarization was that the file entries of the + release tar all have a `./` prefix in the path. This causes a number of + issues, not least of which is that our Gradle extract tasks end up + copying an empty fileset to the destination directory. The workaround + here is imply to remove the leading `./` path segment from each file + when performing the extraction. For more details see this issue: + https://github.com/elastic/elasticsearch/issues/49417 + +commit c9eba175178cc2a2e66cf69dafcbfcfaf397ee59 +Author: jesinity +Date: Fri Nov 22 21:35:09 2019 +0100 + + Fix HLRC parsing of CancelTasks response (#47017) + + Adds support for proper cancel tasks parsing. + + Closes #45414 + +commit 2ec047db04b092b4e988e8a34aa9740c80c2399a +Author: debadair +Date: Wed Nov 13 18:30:11 2019 -0800 + + [DOCS] Rename auditing topic. Closes #49012 (#49013) + + * [DOCS] Rename auditing topic. Closes #49012 + + * Fixed file name, fixed settings link. + + * Add link to settings + +commit d06c71eb8206490e819da270e46d5a982dd57700 +Author: James Rodewig +Date: Fri Nov 22 15:50:27 2019 -0500 + + [DOCS] Fix edge n-gram tokenizer nav + + Adds a missing float tag to the edge n-gram tokenizer docs. This tag + ensures the edge n-gram tokenizer docs display on the same page. + +commit 8eaee7cbdc6a38fc3e6460679d79ea28811287a4 +Author: Dimitris Athanasiou +Date: Fri Nov 22 22:06:10 2019 +0200 + + [7.x][ML] Explain data frame analytics API (#49455) (#49504) + + This commit replaces the _estimate_memory_usage API with + a new API, the _explain API. + + The API consolidates information that is useful before + creating a data frame analytics job. + + It includes: + + - memory estimation + - field selection explanation + + Memory estimation is moved here from what was previously + calculated in the _estimate_memory_usage API. + + Field selection is a new feature that explains to the user + whether each available field was selected to be included or + not in the analysis. In the case it was not included, it also + explains the reason why. + + Backport of #49455 + +commit 69f570ea5f7216c4884c7428f60b862eaa1a9f89 +Author: Jason Tedor +Date: Fri Nov 22 14:42:23 2019 -0500 + + Adjust version on final pipeline serialization + + This commit adjusts the version final pipeline serialization after it + was backported to the 7.5 branch. + +commit 4fd5fb5297989521f037fcc9190b3aecdc384438 +Author: Jay Modi +Date: Fri Nov 22 12:41:52 2019 -0700 + + Stop NodeTests from timing out in certain cases (#49202) (#49503) + + The NodeTests class contains tests that check behavior when shutting + down a node. This involves starting a node, performing some operation, + stopping the node, and then awaiting the close of the node. Part of + closing a node is the termination of the node's ThreadPool. ThreadPool + termination semantics can be deceiving. The ThreadPool#terminate method + takes a timeout value and the first oddity is that the terminate method + can take two times the timeout value before returning. Internally this + method acts on the ExecutorService instances that are held by the + ThreadPool. First, an orderly shutdown is attempted and pending tasks + are allowed to execute while waiting for the timeout value. If any of + the ExecutorService instances have not terminated, a call is made to + attempt to stop all active tasks (usually using interrupts) and then + waits for up to the timeout value a second time for the termination of + the ExecutorService instances. This means that if use a large value + when waiting for a node to close, we may not attempt to interrupt any + threads that are in a blocking call before the test times out. + + In order to avoid causing these tests to time out, this change reduces + the timeout passed to Node#awaitClose to 10 seconds from 1 day. This + will allow blocked threads to be interrupted before the test suite + fails due to the timeout. + + Closes #44256 + Closes #42350 + Closes #44435 + +commit 71bcfbf1e3b080eb480576a134aff61270706630 +Author: Jason Tedor +Date: Fri Nov 22 14:00:38 2019 -0500 + + Replace required pipeline with final pipeline (#49470) + + This commit enhances the required pipeline functionality by changing it + so that default/request pipelines can also be executed, but the required + pipeline is always executed last. This gives users the flexibility to + execute their own indexing pipelines, but also ensure that any required + pipelines are also executed. Since such pipelines are executed last, we + change the name of required pipelines to final pipelines. + +commit 1431c2b408bba1d8adda7c27dbb61f298c1005a3 +Author: Jay Modi +Date: Fri Nov 22 11:59:46 2019 -0700 + + Run build-tools test with Gradle jdk (#49459) (#49497) + + The test task is configured to use the runtime java version, but there + are issues with the version of groovy used by gradle pre 6.0. In order + to workaround this, we use the Gradle JDK to execute the build-tools + tests. + + Closes #49404 + Closes #49253 + +commit 0c4491964bca759b18a4b97773f0edad2143fe9c +Author: Marios Trivyzas +Date: Fri Nov 22 15:28:32 2019 +0100 + + SQL: Fix issue with folding of CASE/IIF (#49449) + + Add extra checks to prevent ConstantFolding rule to try to fold + the CASE/IIF functions early before the SimplifyCase rule gets applied. + + Fixes: #49387 + + (cherry picked from commit f35c9725350e35985d8dd3001870084e1784a5ca) + +commit 49bb5fb64274b3b1a8d709b8d9d847cfa4e6f8f6 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Nov 22 18:12:53 2019 +0100 + + Netty4: switch to composite cumulator (#49478) + + The default merge cumulator used in netty transport leads to additional + GC pressure and memory copying when a message that exceeds the chunk + size is handled. This is especially a problem on G1 GC, since we get + many "humongous" allocations and that can in theory cause real memory + circuit breaker to break unnecessarily. + +commit ca895d3ad5f7c2df200ce5e1f634af1d6e98679e +Author: Lisa Cawley +Date: Fri Nov 22 08:31:30 2019 -0800 + + [DOCS] Merge rollup config details into API (#49412) + +commit 97c7ea60b938467ee2b13970d1ab6895229d5bf4 +Author: Armin Braun +Date: Fri Nov 22 17:27:27 2019 +0100 + + Add Missing Nullable Assertions in SnapshotsService (#49465) (#49492) + + Just realized we were missing some annotations here which was somewhat + confusing since other methods/parameters have the `Nullable` annotation + wherever a `null` can be passed. + +commit 562607d3f554666740e89b61750f5f78c7ba1b94 +Author: James Rodewig +Date: Fri Nov 22 10:38:01 2019 -0500 + + [DOCS] Reformat n-gram token filter docs (#49438) + + Reformats the edge n-gram and n-gram token filter docs. Changes include: + + * Adds title abbreviations + * Updates the descriptions and adds Lucene links + * Reformats parameter definitions + * Adds analyze and custom analyzer snippets + * Adds notes explaining differences between the edge n-gram and n-gram + filters + + Additional changes: + * Switches titles to use "n-gram" throughout. + * Fixes a typo in the edge n-gram tokenizer docs + * Adds an explicit anchor for the `index.max_ngram_diff` setting + +commit 4fae2bb3b128c29b57d04a8f0a3ce5544b8e8b9d +Author: Rory Hunter +Date: Fri Nov 22 14:58:17 2019 +0000 + + Don't close stderr under `--quiet` (#49431) + + Backport of #47208. + + Closes #46900. When running ES with `--quiet`, if ES then exits abnormally, a + user has to go hunting in the logs for the error. Instead, never close + System.err, and print more information to it if ES encounters a fatal error + e.g. config validation, or some fatal runtime exception. This is useful when + running under e.g. systemd, since the error will go into the journal. + + Note that stderr is still closed in daemon (`-d`) mode. + +commit ed787d06e81e9a1a6da3e5e4ed5ea29fc7d60c51 +Author: Benjamin Trent +Date: Fri Nov 22 09:24:06 2019 -0500 + + [7.x] [ML][Inference][HLRC] GET trained models (#49464) (#49488) + + * [ML][Inference][HLRC] GET trained models (#49464) + + * fixing for backport + +commit 12c2ca8895a11a5737700e6f3d2c5b998bc0909b +Author: Enrico Zimuel +Date: Fri Nov 22 15:06:47 2019 +0100 + + Fix missing slash in specification for indices.put_mapping + +commit 56d97dcb6cc0fec4ed12ad93070957d015af9ec0 +Author: István Zoltán Szabó +Date: Fri Nov 22 14:17:44 2019 +0100 + + [DOCS] Replaces deprecated ScriptService.ScriptType.INLINE with supported script in Java update docs. (#49424) + +commit 276b6c67f401aac1089f04bd54e28037428c99be +Author: Benjamin Trent +Date: Fri Nov 22 08:14:33 2019 -0500 + + [ML][Inference] Fixing pre-processor value handling and size estimate (#49270) (#49489) + + * [ML][Inference] Fixing pre-processor value handling and size estimate + + * fixing npe + +commit 35cc0e094826fbce3cf25b21609e792cff3ed641 +Author: István Zoltán Szabó +Date: Fri Nov 22 11:15:35 2019 +0100 + + [DOCS] Removes the default size definition of thread pool types (#49442) + + Co-Authored-By: James Rodewig + +commit d444c334d773b59bbc64e08c86ce1e18cc01b9cf +Author: Florian Kelbert +Date: Fri Nov 22 10:55:08 2019 +0100 + + Modify example for pinned query (#49481) + + I do not see any reason to advertise phones of specific companies. + +commit ed4eecc00e266fbc47cd5ae63d8b3d8a483a620b +Author: Jim Ferenczi +Date: Thu Nov 21 22:45:52 2019 +0100 + + Pre-sort shards based on the max/min value of the primary sort field (#49092) + + This change automatically pre-sort search shards on search requests that use a primary sort based on the value of a field. When possible, the can_match phase will extract the min/max (depending on the provided sort order) values of each shard and use it to pre-sort the shards prior to running the subsequent phases. This feature can be useful to ensure that shards that contain recent data are executed first so that intermediate merge have more chance to contain contiguous data (think of date_histogram for instance) but it could also be used in a follow up to early terminate sorted top-hits queries that don't require the total hit count. The latter could significantly speed up the retrieval of the most/least + recent documents from time-based indices. + + Relates #49091 + +commit c13fce60a8a2a2023fe84125bd7296b946badac7 +Author: István Zoltán Szabó +Date: Fri Nov 22 09:13:55 2019 +0100 + + [DOCS] Removes data frame leftovers from transforms overview (#49434) + +commit d42eac9cf392a0de9fda454ceb8afee3b4b7f822 +Author: Przemyslaw Gomulka +Date: Fri Nov 22 09:54:34 2019 +0100 + + [DOC] Modify the update example to change a document (#49228) (#49443) + + Example at the moment is not changing the existing document. Update request should at least modify the existing document. + +commit 1fbb248cb7193f2234b3b17ba77421db2a9079e9 +Author: Hendrik Muhs +Date: Fri Nov 22 08:50:10 2019 +0100 + + reenable warning checks in pivot tests (#49436) + +commit 224374345071211fc16369a548cbafb9917aad32 +Author: Martijn van Groningen +Date: Fri Nov 22 08:38:29 2019 +0100 + + Update geolite2 database in ingest geoip plugin. (#49308) + + Some tests were tweaked to deal with the updated database files. + +commit c239a6a493e1c3491649212faede1aa1eb311c76 +Author: Mark Vieira +Date: Thu Nov 21 17:44:21 2019 -0800 + + Fix build failure when attempting to export UBI Docker image (#49472) + +commit 2e5f2dd1e1539790ac5a851367ef2cc000d4d5bf +Author: Tim Vernum +Date: Fri Nov 22 12:14:55 2019 +1100 + + Deprecate misconfigured SSL server config (#49280) + + This commit adds a deprecation warning when starting + a node where either of the server contexts + (xpack.security.transport.ssl and xpack.security.http.ssl) + meet either of these conditions: + + 1. The server lacks a certificate/key pair (i.e. neither + ssl.keystore.path not ssl.certificate are configured) + 2. The server has some ssl configuration, but ssl.enabled is not + specified. This new validation does not care whether ssl.enabled is + true or false (though other validation might), it simply makes it + an error to configure server SSL without being explicit about + whether to enable that configuration. + + Backport of: #45892 + +commit a7477ad7c391be44f1758615a40d5285a0931c05 +Author: Benjamin Trent +Date: Thu Nov 21 15:32:32 2019 -0500 + + [7.x] [ML][Inference] compressing model definition and lazy parsing (#49269) (#49446) + + * [ML][Inference] compressing model definition and lazy parsing (#49269) + + * [ML][Inference] compressing model definition and lazy parsing + + * addressing PR comments + + * adding commons io + + * implementing simplified bounded stream + + * adjusting for type inclusion + +commit e8971ff36792cb53247d4d2930cbf3034049349b +Author: Igor Motov +Date: Thu Nov 21 11:12:02 2019 -0500 + + Geo: Fix handling of circles in legacy geo_shape queries (#49410) + + Brings back support for circles in legacy geo_shape queries that + was accidentally lost during query refactoring. + + Fixes #49296 + +commit 231d079bf855c7dab23e2148f1e44c527ca392bb +Author: Armin Braun +Date: Thu Nov 21 19:57:50 2019 +0100 + + Fix Azure Mock Issues (#49377) (#49381) + + Fixing a few small issues found in this code: + 1. We weren't reading the request headers but the response headers when checking for blob existence in the mocked single upload path + 2. Error code can never be `null` removed the dead code that resulted + 3. In the logging wrapper we weren't checking for `Throwable` so any failing assertions in the http mock would not show up since they + run on a thread managed by the mock http server + +commit 0fa3b887b7d0961aed02b851f5591f6da04989b5 +Author: James Rodewig +Date: Thu Nov 21 13:05:53 2019 -0500 + + [DOCS] Document several missing thread pools (#48543) + + Adds documentation for the following thread pools: + - fetch_shard_started + - fetch_shard_store + - flush + - force_merge + - management + + Closes #48524 + + Co-Authored-By: Jay Modi + +commit af183e2ebb61109219822b67591294a6c0953dfe +Author: Tal Levy +Date: Thu Nov 21 09:02:30 2019 -0800 + + correct licensing and incorporation of FastMath (#49122) (#49441) + + this resolves incorrectly licensed code in #49009. + + ESSloppyMath is made as a wrapper around FastMath.java which is + not meant to be modified with code beyond the original source + +commit 420825c3b5fbf04bedc4132d4177ece95a69585c +Author: Yannick Welsch +Date: Thu Nov 21 17:37:41 2019 +0100 + + Strengthen validateClusterFormed check (#49248) + + Strengthens the validateClusterFormed check that is used by the test infrastructure to make + sure that nodes are properly connected and know about each other. Is used in situations where + the cluster is scaled up and down, and where there previously was a network disruption that has + been healed. + + Closes #49243 + +commit d9835f7fb4443a2baff05fc895dbd9fbc7a91353 +Author: Benjamin Trent +Date: Thu Nov 21 11:22:16 2019 -0500 + + [ML] Fix r_squared eval when variance is 0 (#49439) (#49445) + +commit 138d16ab9edfb2abf458a01a29059233741ff7e7 +Author: Christoph Büscher +Date: Thu Nov 21 17:14:01 2019 +0100 + + Fix ClusterHealthResponsesTests condition (#49360) + + Currently the condtion that is supposed to test creation of test instances with + multiple indices is never true because it compares Strings with an enum. This + changes it so the condition uses the enum constants instead. + +commit 779b4bd92b19378d7aeaee1d2a543fb0c8b9b25e +Author: Hendrik Muhs +Date: Thu Nov 21 16:14:41 2019 +0100 + + update the name of the audit index (#49432) + + small update to the name of the audit index changed in 7.5 + +commit d41b2e3f38eecb7ba7405e8a028e145419ce8f22 +Author: Benjamin Trent +Date: Thu Nov 21 09:46:34 2019 -0500 + + [ML][Inference] allowing per-model licensing (#49398) (#49435) + + * [ML][Inference] allowing per-model licensing + + * changing to internal action + removing pre-mature opt + +commit f264808a6a0284022eac3fc2882673294074c900 +Author: James Rodewig +Date: Thu Nov 21 09:05:33 2019 -0500 + + [DOCS] Replace cross-cluster search PNG images with SVGs (#49395) + +commit c7ac2011eb50a35988a81cfef826f7c9a861d836 +Author: Przemysław Witek +Date: Thu Nov 21 15:01:18 2019 +0100 + + [7.x] Implement accuracy metric for multiclass classification (#47772) (#49430) + +commit 03600e4e1253777b2a59131a3213cca5e99bc554 +Author: James Rodewig +Date: Thu Nov 21 08:53:56 2019 -0500 + + [DOCS] Document `script_score` float precision limit (#49402) + + All document scores are positive 32-bit floating point numbers. However, this + wasn't previously documented. + + This can result in surprising behavior, such as precision loss, for users when + customizing scores using the function score query. + + This commit updates an existing admonition in the function score query docs to + document the 32-bits precision limit. It also updates the search API reference + docs to note that `_score` is a 32-bit float. + +commit 3eb577f6c87e336adb65346abc04729e5db27dbe +Author: weizijun +Date: Thu Nov 21 21:29:45 2019 +0800 + + Document all shard allocation filtering attributes (#46992) + + This commit adds coverage to the docs for some missing built-in shard + allocation attributes. + +commit d59ea64ccd9d0f6eaa7de3c3e99b8a9a283c83cd +Author: Martijn van Groningen +Date: Thu Nov 21 14:22:41 2019 +0100 + + Monitoring should wait with collecting data when cluster service is started. (#49426) + + Backport of #48277 + + Otherwise integration tests may fail if the monitoring interval is low: + ``` + [2019-10-21T09:57:25,527][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [integTest-0] fatal error in thread [elasticsearch[integTest-0][generic][T#4]], exiting + java.lang.AssertionError: initial cluster state not set yet + at org.elasticsearch.cluster.service.ClusterApplierService.state(ClusterApplierService.java:208) ~[elasticsearch-7.6.0-SNAPSHOT.jar:7.6.0-SNAPSHOT] + at org.elasticsearch.cluster.service.ClusterService.state(ClusterService.java:125) ~[elasticsearch-7.6.0-SNAPSHOT.jar:7.6.0-SNAPSHOT] + at org.elasticsearch.xpack.monitoring.MonitoringService$MonitoringExecution$1.doRun(MonitoringService.java:231) ~[?:?] + at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.6.0-SNAPSHOT.jar:7.6.0-SNAPSHOT] + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?] + at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] + at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.6.0-SNAPSHOT.jar:7.6.0-SNAPSHOT] + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] + at java.lang.Thread.run(Thread.java:835) [?:?] + ``` + + I ran into this when lowering the monitoring interval when investigating + enrich monitoring test: #48258 + +commit c3e4405ddf33b488fb121965183a0ce059a4cd92 +Author: Hendrik Muhs +Date: Thu Nov 21 13:52:14 2019 +0100 + + [7.x][Transform] Transform fix force stop race condition (#49249) (#49420) + + fix force stopping transform if indexer state hasn't been written and/or is set to STOPPED. In certain situations the transform could not be stopped, which means the task could not be removed. Introduces improved abstraction in order to better test state handling in future. + +commit 010c3de47ef56a45a46465e979adabce9632d821 +Author: Andrei Dan +Date: Thu Nov 21 11:41:32 2019 +0000 + + Slm set operation mode to RUNNING on first run (#49236) (#49425) + + * SLM set the operation mode to RUNNING on first run + + Set the SLM operation mode to RUNNING when setting the first SLM lifecycle + policy. Historically, SLM was not decoupled from ILM but now they are + independent components. Setting the SLM operation mode to what the ILM running + mode was when we set the first SLM lifecycle policy was a remain from those + times. + + * SLM update package info + + * SLM suppress unusued warning + + * SLM use logger for the correct class + + * SLM Add integration test for operation mode + + * Use ESSingleNodeTestCase instead of ESIntegTestCase + + (cherry picked from commit 4ad3d93f89d03bf9a25685a990d1a439f33ce0e6) + Signed-off-by: Andrei Dan + +commit 0164de8579fa839df527e01186b70680abcc3047 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Nov 21 11:44:27 2019 +0100 + + Reindex search response fix again (#49423) + + Fixed test case to more broadly accept all messages with "Partial + shards failure" in it, to hopefully catch all relevant search messages + now that reindex does not allow searching against red shards. + + Closes #49295 + +commit df8d7b213bc1d215a4be0bfa6549c7f5cc79f381 +Author: Armin Braun +Date: Thu Nov 21 11:29:26 2019 +0100 + + Add Logging to Mock Repo API Server (#49409) + + While we log exception in the handler, we may still miss exceptions + hgiher up the execution chain. This adds logging of exceptions to all + operations on the IO loop including connection establishment. + + Relates #49401 + +commit 3221827a4b5a6fea51803d16ec9a81addf8fc72b +Author: Peter Johnson +Date: Thu Nov 21 11:30:15 2019 +0100 + + [Docs] Correct typo in match-query.asciidoc (#49082) + +commit d1eb7e749ec927e27a06badf0c3274d10c519433 +Author: Alan Woodward +Date: Thu Nov 21 09:49:23 2019 +0000 + + Fix test for index phrases shortcut with multi-term synonyms (#49366) + + Lucene 8.3 included a root fix for #43976, which was temporarily fixed in elasticsearch + by #44340. Since we have upgraded to 8.3 we no longer need this workaround. This + commit fixes the test that was added to check the workaround, and instead checks that + fields with index_phrases enabled correctly build queries when used with multi-term + synonyms. + + Closes #47777 + +commit d72bd3a171d5624dacc32f3e83a47bc49fcbef99 +Author: Yannick Welsch +Date: Thu Nov 21 10:03:11 2019 +0100 + + Verify translog checksum before UUID check (#49394) + + When opening a translog file, we check whether the UUID matches what we expect (the UUID + from the latest commit). The UUID check can in certain cases fail when the translog is + corrupted. This commit changes the ordering of the checks so that corruption is detected first. + +commit 8ee70fa9c6805233968f53e7995eb0ce0889cb2e +Author: Yannick Welsch +Date: Thu Nov 21 10:03:25 2019 +0100 + + Fix testPeerRecoveryTrimsLocalTranslog (#49385) + + 7.x uses the transport client, which, when being closed, can throw an IllegalStateException + + Closes #49375 + +commit 5b10fd301ea4a23fe6d10e95b815eccba8af8489 +Author: István Zoltán Szabó +Date: Thu Nov 21 09:51:35 2019 +0100 + + [DOCS] Fixes endpoint schema in PUT app privileges API docs. (#49390) + +commit 61c54fd617bd1eaf11403c43a0271c7cf8801314 +Author: Lisa Cawley +Date: Wed Nov 20 16:41:58 2019 -0800 + + [DOCS] Qualifies Watcher transforms (#47482) + +commit 37a9cd677b7ea6b934a81f6d104530a5ef6f8f05 +Author: Nhat Nguyen +Date: Mon Nov 18 11:29:18 2019 -0500 + + Ignore Lucene index in peer recovery if translog corrupted (#49114) + + If the translog on a replica is corrupt, we should not perform an + operation-based recovery or utilize sync_id as we won't be able to open + an engine in the next step. This change adds an extra validation that + ensures translog is okay when preparing a peer recovery request. + +commit fec22130c287c6d96bdff8930b545e965f194159 +Author: Nhat Nguyen +Date: Mon Nov 11 08:57:46 2019 -0500 + + Improve error message when pausing index (#48915) + + Throw an appropriate error message when the follower index is not found + or is a regular index. + +commit d3bc9b7fb2a4ecd05078c4419cfb927db94b1617 +Author: debadair +Date: Wed Nov 20 10:54:09 2019 -0800 + + [DOCS] Clarify backport policy for important technical corrections. (#49131) + + * [DOCS] Clarify backport policy for important technical corrections. + + * Update docs/README.asciidoc + + Co-Authored-By: James Rodewig + +commit 0f157366878c1ecd2f72770fafeca12e130129ea +Author: Lisa Cawley +Date: Wed Nov 20 10:43:53 2019 -0800 + + [DOCS] Reformat rollup API docs (#49397) + +commit d9fd4cc351292b2cd7583e3381da751741305456 +Author: jaymode +Date: Wed Nov 20 10:16:35 2019 -0700 + + Add version 6.8.6 + +commit 06c268980209e9d3cbd62da394d861d3e26f2a2f +Author: Hendrik Muhs +Date: Wed Nov 20 18:51:11 2019 +0100 + + rename data frame tests to transform tests (#49361) + + rename files and tests in rolling upgrade tests to transform + +commit 8c2ab8bb72135299807f7c5adfb2894386b2181b +Author: Bogdan Pintea +Date: Wed Nov 20 16:53:16 2019 +0100 + + SQL:Docs: add the PIVOT clause to SELECT section (#49129) + + The PR adds the documentation on the PIVOT clause. + + (cherry picked from commit a55b36065e6496c44b6e3191296931d477a8e5f5) + +commit a27e0fe10de9d3fe746913b68e4c1e4cbc60d91c +Author: Lisa Cawley +Date: Wed Nov 20 08:19:33 2019 -0800 + + [DOCS] Reformat ILM API docs (#49348) + +commit a780ec14f05ed7affe606f211c44be4bb1ae7004 +Author: Jack Conradson +Date: Wed Nov 20 07:07:33 2019 -0800 + + Painless: Upgrade ASM to 7.2 (#49263) + + This upgrades Painless to use the latest ASM libraries providing support up + to Java 14. Note the library is not published with the latest versions in an + "all" package, so we pick up each lib independently that's required. There + were some changes to the getType method that require descriptors to be + used in place of internal class names. + +commit 81548df2d90a1ff291267656ca353e17e61d382f +Author: Jim Ferenczi +Date: Wed Nov 20 15:33:41 2019 +0100 + + Disable caching when queries are profiled (#48195) + + This change disables the query and request cache when + profile is set to true in the request. This means that profiled queries + will not check caches to execute the query and the result will never be + added in the cache either. + + Closes #33298 + +commit 1cde4a6364ce26ab38c99d2bc08aa4a1d4ce3be6 +Author: Armin Braun +Date: Wed Nov 20 15:22:10 2019 +0100 + + Make SnapshotsService#getRepositoryData Async (#49322) (#49358) + + * Make SnapshotsService#getRepositoryData Async (#49322) + + Follow up to #49299 removing the blocking step for the + snapshot status APIs as well. + +commit 20558cf61c42853c35e9b755b3d386e9b2d78ac9 +Author: David Roberts +Date: Wed Nov 20 12:36:15 2019 +0000 + + [ML] Fix simultaneous stop and force stop datafeed (#49367) + + If a datafeed is stopped normally and force stopped at the same + time then it is possible that the force stop removes the + persistent task while the normal stop is performing actions. + Currently this causes the normal stop to error, but since + stopping a stopped datafeed is not an error this doesn't make + sense. Instead the force stop should just take precedence. + + This is a followup to #49191 and should really have been + included in the changes in that PR. + +commit e3da60c23d17fb983000ff78d6670a56a5450514 +Author: Mayya Sharipova +Date: Wed Nov 20 07:43:14 2019 -0500 + + Increase the number of vector dims to 2048 (#46895) + +commit 6bad28a835362f98f4cc971b89dc6be06675fdf6 +Author: Tanguy Leroux +Date: Wed Nov 20 05:13:56 2019 -0500 + + Mute AzureBlobStoreRepositoryTests (#49364) + + Relates #48978 + +commit 4ffa050735ad7bce2b8574eceaab8bb0f3098aaa +Author: Christoph Büscher +Date: Wed Nov 20 10:36:39 2019 +0100 + + Allow custom characters in token_chars of ngram tokenizers (#49250) + + Currently the `token_chars` setting in both `edgeNGram` and `ngram` tokenizers + only allows for a list of predefined character classes, which might not fit + every use case. For example, including underscore "_" in a token would currently + require the `punctuation` class which comes with a lot of other characters. + This change adds an additional "custom" option to the `token_chars` setting, + which requires an additional `custom_token_chars` setting to be present and + which will be interpreted as a set of characters to inlcude into a token. + + Closes #25894 + +commit c6b31162ba152547cdfd9c7dc8074758a69adc78 +Author: Alan Woodward +Date: Wed Nov 20 09:21:01 2019 +0000 + + Refactor percolator's QueryAnalyzer to use QueryVisitors + + Lucene now allows us to explore the structure of a query using QueryVisitors, + delegating the knowledge of how to recurse through and collect terms to the + query implementations themselves. The percolator currently has a home-grown + external version of this API to construct sets of matching terms that must be + present in a document in order for it to possibly match the query. + + This commit removes the home-grown implementation in favour of one using + QueryVisitor. This has the added benefit of making interval queries available + for percolator pre-filtering. Due to a bug in multi-term intervals (LUCENE-9050) + it also includes a clone of some of the lucene intervals logic, that can be removed + once upstream has been fixed. + + Closes #45639 + +commit 9c0ec7ce23f863206a7743dc54b0bf169ac759af +Author: Przemysław Witek +Date: Wed Nov 20 10:08:16 2019 +0100 + + [7.x] Make AnalyticsProcessManager class more robust (#49282) (#49356) + +commit f753fa22656e525fd9723a1412505328a35254c0 +Author: Tanguy Leroux +Date: Tue Nov 19 09:55:36 2019 -0500 + + HttpHandlers should return correct list of objects (#49283) + + This commit fixes the server side logic of "List Objects" operations + of Azure and S3 fixtures. Until today, the fixtures were returning a " + flat" view of stored objects and were not correctly handling the + delimiter parameter. This causes some objects listing to be wrongly + interpreted by the snapshot deletion logic in Elasticsearch which + relies on the ability to list child containers of BlobContainer (#42653) + to correctly delete stale indices. + + As a consequence, the blobs were not correctly deleted from the + emulated storage service and stayed in heap until they got garbage + collected, causing CI failures like #48978. + + This commit fixes the server side logic of Azure and S3 fixture when + listing objects so that it now return correct common blob prefixes as + expected by the snapshot deletion process. It also adds an after-test + check to ensure that tests leave the repository empty (besides the + root index files). + + Closes #48978 + +commit 4d6e037e9053baae71b3eaa40321c02759d4b6d8 +Author: Dimitris Athanasiou +Date: Wed Nov 20 10:02:42 2019 +0200 + + [7.x][ML] Extract creation of DFA field extractor into a factory (#49315) (#49329) + + This commit moves the async calls required to retrieve the components + that make up `ExtractedFieldsExtractor` out of `DataFrameDataExtractorFactory` + and into a dedicated `ExtractorFieldsExtractorFactory` class. + + A few more refactorings are performed: + + - The detector no longer needs the results field. Instead, it knows + whether to use it or not based on whether the task is restarting. + - We pass more accurately whether the task is restarting or not. + - The validation of whether fields that have a cardinality limit + are valid is now performed in the detector after retrieving the + respective cardinalities. + + Backport of #49315 + +commit 543f5f4fafdbc49084e64209bf28b331f23bbb83 +Author: Dimitris Athanasiou +Date: Wed Nov 20 09:58:13 2019 +0200 + + [7.x][ML][HLRC] Add FAILED state for data frame analytics (#49326) (#49327) + + Backport of #49326 + +commit 92a1faf545e2af59009d8f582c9a8945f0184512 +Author: Mathew Davis <52668540+matt-davis-elastic@users.noreply.github.com> +Date: Tue Nov 19 21:36:16 2019 -0700 + + Fixing a typo in the stop SLM api request header. + +commit 2b9fb7ebe2bca563aa982c95e4cbfa3c5ef2882b +Author: Lisa Cawley +Date: Tue Nov 19 16:08:46 2019 -0800 + + [DOCS] Merges security overview pages (#49342) + +commit 42bb8ae5254b99bb29ffd7e73409150822b02da6 +Author: Przemysław Witek +Date: Tue Nov 19 22:47:12 2019 +0100 + + [7.x] Extract indexData method out of RegressionIT tests (#49306) (#49313) + +commit 17358b5af7b34a2b9a15846ba672f5a04001fcd5 +Author: Mark Tozzi +Date: Tue Nov 19 16:44:29 2019 -0500 + + (refactor) Extract Empty/Script/Missing ValuesSource behavior to an interface (#48320) (#49330) + + This is a pure code rearrangement refactor. Logic for what specific ValuesSource instance to use for a given type (e.g. script or field) moved out of ValuesSourceConfig and into CoreValuesSourceType (previously just ValueSourceType; we extract an interface for future extensibility). ValueSourceConfig still selects which case to use, and then the ValuesSourceType instance knows how to construct the ValuesSource for that case. + +commit d068818b16723589bc34010fa0fdc7d2b8bcca6a +Author: Benjamin Trent +Date: Tue Nov 19 16:43:19 2019 -0500 + + [ML][Inference] document new settings (#49309) (#49336) + + * [ML][Inference] document new settings + + * [DOCS] Minor edits + +commit 62a3154d0ed3506a83e40959774cd92fa32c9b76 +Author: James Rodewig +Date: Tue Nov 19 16:38:13 2019 -0500 + + [DOCS] [7.x] Add high-level docs for enrich processor and policies (#49194) (#49331) + +commit 75f1f612c263e6289c16e02896b04f28ea143c6a +Author: Lisa Cawley +Date: Tue Nov 19 13:05:11 2019 -0800 + + [DOCS] Merges duplicate pages for Active Directory realms (#49205) + +commit eed4cd25ebcccfc0821d4ef6a6ba60c66df0b4b4 +Author: Jay Modi +Date: Tue Nov 19 13:15:16 2019 -0700 + + ThreadPool and ThreadContext are not closeable (#43249) (#49273) + + This commit changes the ThreadContext to just use a regular ThreadLocal + over the lucene CloseableThreadLocal. The CloseableThreadLocal solves + issues with ThreadLocals that are no longer needed during runtime but + in the case of the ThreadContext, we need it for the runtime of the + node and it is typically not closed until the node closes, so we miss + out on the benefits that this class provides. + + Additionally by removing the close logic, we simplify code in other + places that deal with exceptions and tracking to see if it happens when + the node is closing. + + Closes #42577 + +commit c4c8a7a43c66f8b2f6a557d1d17eb039770386ff +Author: Lisa Cawley +Date: Tue Nov 19 10:29:20 2019 -0800 + + [DOCS] Merges duplicate pages for PKI realms (#49206) + +commit c6a8913c38d98f5b99962e35993e127a4bcc6592 +Author: Ryan Ernst +Date: Tue Nov 19 10:18:25 2019 -0800 + + Fix java home validation usage by tasks (#49204) + + Tasks intending to use a particular java home provided by JAVA_HOME + use the getJavaHome method, which verifies the given java home is + available, or will be if the task will run. However, the verification + logic was broken, in addition to unnecessarily delaying retrieving the + java home until runtime. This commit fixes the verification logic to run + at either config time, delaying verification, or at runtime which + immediately checks if java home is available. + + closes #49153 + +commit 14d2e795aece5ab30d07d68adee01e9e3ec6099b +Author: Jack Conradson +Date: Tue Nov 19 08:32:15 2019 -0800 + + make dim files mmapped (#49272) + + This change mmaps dim files in HybridDirectory to take advantage of off- + heap BKD trees. This is based off of (#48509) via + (https://issues.apache.org/jira/browse/LUCENE-8932). + +commit 2f5acae4a9cb8ef8248cc73ab9e1e0ea1fcef36f +Author: Lisa Cawley +Date: Tue Nov 19 09:54:40 2019 -0800 + + [DOCS] Groups pages related to encrypting communications (#49324) + +commit 62bbe419d3d7a09512d2936dbfe553ba6c1bfddb +Author: Lisa Cawley +Date: Tue Nov 19 09:08:36 2019 -0800 + + [DOCS] Removes Beats security page (#49276) + +commit 19780e20ba83683f7e458a70954c7064ea2bc02b +Author: Andrei Dan +Date: Tue Nov 19 17:14:46 2019 +0000 + + Handle failure to retrieve ILM policy step better (#49193) (#49316) + + This commit wraps the calls to retrieve the current step in a try/catch + so that the exception does not bubble up. Instead, step info is added + containing the exception to the existing step. + + Semi-related to #49128 + + (cherry picked from commit 72530f8a7f40ae1fca3704effb38cf92daf29057) + Signed-off-by: Andrei Dan + +commit 97cdfd2848ad70b5bc7ab576218a22bb06cec4b0 +Author: Lisa Cawley +Date: Tue Nov 19 08:31:24 2019 -0800 + + [DOCS] Clarify ML job closure prerequisites (#49265) + +commit a26916cc23c7c1fa1fe5f49350d19628fd5e46b9 +Author: James Rodewig +Date: Tue Nov 19 10:54:29 2019 -0500 + + [DOCS] Reformat elision token filter docs (#49262) + +commit 8639ddab5e1c7ffde8bee16bd32db54c9c11b68c +Author: James Rodewig +Date: Tue Nov 19 10:54:16 2019 -0500 + + [DOCS] Reformat fingerprint token filter docs (#49311) + +commit 0acba44a2e8e22ae2899b4b267837e19ff525f11 +Author: Armin Braun +Date: Tue Nov 19 16:49:12 2019 +0100 + + Make Repository.getRepositoryData an Async API (#49299) (#49312) + + This API call in most implementations is fairly IO heavy and slow + so it is more natural to be async in the first place. + Concretely though, this change is a prerequisite of #49060 since + determining the repository generation from the cluster state + introduces situations where this call would have to wait for other + operations to finish. Doing so in a blocking manner would break + `SnapshotResiliencyTests` and waste a thread. + Also, this sets up the possibility to in the future make use of async IO + where provided by the underlying Repository implementation. + + In a follow-up `SnapshotsService#getRepositoryData` will be made async + as well (did not do it here, since it's another huge change to do so). + Note: This change for now does not alter the threading behaviour in any way (since `Repository#getRepositoryData` isn't forking) and is purely mechanical. + +commit cb5169ae377103ef8a82bec3ac7c353c373760c9 +Author: jimczi +Date: Tue Nov 19 16:14:07 2019 +0100 + + update release notes for 7.5.0 after respin + +commit bc29c9877a2aaa78e31955130152935cf508d4d6 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Nov 19 14:37:03 2019 +0100 + + Reindex search response fix (#49301) + + Fixed test case to also accept another error message, now that reindex + does not allow searching against red shards. + + Closes #49295 + +commit fd1bb4a33a54edb47074ddb3276d51a9f34f6028 +Author: Marios Trivyzas +Date: Tue Nov 19 13:40:34 2019 +0100 + + SQL: Fix issue with mins & hours for DATEDIFF (#49252) + + Previously, DATEDIFF for minutes and hours was doing a + rounding calculation using all the time fields (secs, msecs/micros/nanos). + Instead it should first truncate the 2 dates to the respective field (mins or hours) + zeroing out all the more detailed time fields and then make the subtraction. + + (cherry picked from commit 124cd18e20429e19d52fd8dc383827ea5132d428) + +commit 19602fd5731ab5991f2e179f1bfb9c0cfe55d780 +Author: Benjamin Trent +Date: Tue Nov 19 07:56:40 2019 -0500 + + [ML][Inference] changing setting to be memorySizeSettting (#49259) (#49302) + +commit 38aec2e298fb1148bb6e52743d4b9ef11e785774 +Author: Przemysław Witek +Date: Tue Nov 19 12:50:14 2019 +0100 + + Relax assertions related to datafeed timing stats in .yml test (#49285) (#49291) + +commit abed869ec63b90faf0b883a24225af59619e3318 +Author: Tanguy Leroux +Date: Tue Nov 19 06:37:02 2019 -0500 + + Mute ReindexFailureTests.testResponseOnSearchFailure (#49298) + + Relates #49295 + +commit a5204c1c80c7dc2de9c7ff312a2e215c0b7c97e9 +Author: David Roberts +Date: Tue Nov 19 10:51:46 2019 +0000 + + [ML] Fixes for stop datafeed edge cases (#49284) + + The following edge cases were fixed: + + 1. A request to force-stop a stopping datafeed is no longer + ignored. Force-stop is an important recovery mechanism + if normal stop doesn't work for some reason, and needs + to operate on a datafeed in any state other than stopped. + 2. If the node that a datafeed is running on is removed from + the cluster during a normal stop then the stop request is + retried (and will likely succeed on this retry by simply + cancelling the persistent task for the affected datafeed). + 3. If there are multiple simultaneous force-stop requests for + the same datafeed we no longer fail the one that is + processed second. The previous behaviour was wrong as + stopping a stopped datafeed is not an error, so stopping + a datafeed twice simultaneously should not be either. + + Backport of #49191 + +commit 9c006483141cbb8d116461a6b047218f2b390861 +Author: Armin Braun +Date: Tue Nov 19 09:50:52 2019 +0100 + + Make Snapshot Delete Concurrency Exception Consistent (#49266) (#49281) + + We shouldn't be throwing `RepositoryException` when the repository + wasn't concurrently modified in an unexpected fashion (i.e. on the blob/file level). + When we know that the known repo gen moved higher in terms of the generation + tracked in master memory we should throw the concurrent snapshot exception. + + This change makes concurrent snapshot create and delete always throw the same exception, + prevents unnecessary listings when the generation is known to be off and prevents + future test failures in SLM tests that assume the concurrent snapshot exception + is always thrown here. + Without this change, the newly added test randomly fails the `instanceOf` assertion + by running into a `RepositoryException`. + +commit abd4a70b106a88e3c3f46d571eb7fbdc60dbbcc3 +Author: Lisa Cawley +Date: Mon Nov 18 15:19:13 2019 -0800 + + [DOCS] Merges duplicate pages for Kerberos realms (#49207) + +commit b4f82c9cdb617d2dbed33ce96b4b2d2c2812f616 +Author: Lisa Cawley +Date: Mon Nov 18 13:29:51 2019 -0800 + + [DOCS] Merges duplicate pages for LDAP realms (#49203) + +commit 81a9d98a4753cf4dddd7c47c0fae3928d2fd9287 +Author: Julie Tibshirani +Date: Mon Nov 18 11:57:18 2019 -0800 + + Remove the 'experimental' marking from vector fields. (#49120) + + We wrapped up the API changes we wanted to make, and vector fields can now be + considered GA. + +commit a0ee6c8f7ea19b05411e466639380a198fb7bc10 +Author: Julie Tibshirani +Date: Mon Nov 18 12:29:42 2019 -0800 + + Add telemetry for flattened fields. (#48972) (#49125) + + Currently we just record the number of flattened fields defined in the mappings. + +commit 2ac38fd3153ddbdf679c65ed4c1e17644a83d742 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Nov 18 21:22:36 2019 +0100 + + Reindex and friends fail on RED shards (#45830) + + Reindex, update by query and delete by query would silently disregard + RED/unavailable shards, thus not copying, updating or deleting matching + data in those shards. Now use `allow_partial_search_results=false` to + ensure these operations fail if the search crosses an unavailable chard. + + Added the option to explicitly specify `allow_partial_search_results=true` + for reindex only (seemed too strange for update/delete by query). + + Relates #45739 and #42612 + +commit b0054eecd6af79dc5dcb55b7bc1b2675db17ee6e +Author: Lisa Cawley +Date: Mon Nov 18 11:51:02 2019 -0800 + + [DOCS] Merges duplicate pages for file realms (#49200) + +commit eefe7688ced859f891de9a1ce3eae624fcf22dd0 +Author: Benjamin Trent +Date: Mon Nov 18 13:19:17 2019 -0500 + + [7.x][ML] ML Model Inference Ingest Processor (#49052) (#49257) + + * [ML] ML Model Inference Ingest Processor (#49052) + + * [ML][Inference] adds lazy model loader and inference (#47410) + + This adds a couple of things: + + - A model loader service that is accessible via transport calls. This service will load in models and cache them. They will stay loaded until a processor no longer references them + - A Model class and its first sub-class LocalModel. Used to cache model information and run inference. + - Transport action and handler for requests to infer against a local model + Related Feature PRs: + + * [ML][Inference] Adjust inference configuration option API (#47812) + + * [ML][Inference] adds logistic_regression output aggregator (#48075) + + * [ML][Inference] Adding read/del trained models (#47882) + + * [ML][Inference] Adding inference ingest processor (#47859) + + * [ML][Inference] fixing classification inference for ensemble (#48463) + + * [ML][Inference] Adding model memory estimations (#48323) + + * [ML][Inference] adding more options to inference processor (#48545) + + * [ML][Inference] handle string values better in feature extraction (#48584) + + * [ML][Inference] Adding _stats endpoint for inference (#48492) + + * [ML][Inference] add inference processors and trained models to usage (#47869) + + * [ML][Inference] add new flag for optionally including model definition (#48718) + + * [ML][Inference] adding license checks (#49056) + + * [ML][Inference] Adding memory and compute estimates to inference (#48955) + + * fixing version of indexed docs for model inference + +commit 48f53efd9a99442f0ef07288c6607e6815deacee +Author: Lisa Cawley +Date: Mon Nov 18 09:58:47 2019 -0800 + + [DOCS] Merges duplicate pages for SAML realms (#49209) + +commit b0b5fcc4f63118cebc20910aa4286dfbe4e7533b +Author: Lisa Cawley +Date: Mon Nov 18 09:19:11 2019 -0800 + + [DOCS] Removes closed security PRs from release notes (#49256) + +commit 7d20b50f456508bcac559090ef2688b49e250f3e +Author: gpaimla +Date: Mon Nov 18 18:19:54 2019 +0200 + + Implement Lucene EstonianAnalyzer, Stemmer (#49149) + + This PR adds a new analyzer and stemmer for the Estonian language. + + Closes #48895 + +commit 25cc8e36636d9766ba229f2ed1a80991662431a4 +Author: Armin Braun +Date: Mon Nov 18 16:44:09 2019 +0100 + + Fix RepoCleanup not Removed on Master-Failover (#49217) (#49239) + + The logic for `cleanupInProgress()` was backwards everywhere (method itself and + all but one user). Also, we weren't checking it when removing a repository. + + This lead to a bug (in the one spot that didn't use the method backwards) that prevented + the cleanup cluster state entry from ever being removed from the cluster state if master + failed over during the cleanup process. + + This change corrects the backwards logic, adds a test that makes sure the cleanup + is always removed and adds a check that prevents repository removal during cleanup + to the repositories service. + + Also, the failure handling logic in the cleanup action was broken. Repeated invocation would lead to the cleanup being removed from the cluster state even if it was in progress. Fixed by adding a flag that indicates whether or not any removal of the cleanup task from the cluster state must be executed. Sorry for mixing this in here, but I had to fix it in the same PR, as the first test (for master-failover) otherwise would often just delete the blocked cleanup action as a result of a transport master action retry. + +commit e4f6eaeaf52bd5e7acbd50e974ccd9b7b5e13078 +Author: Armin Braun +Date: Mon Nov 18 16:43:47 2019 +0100 + + Fix Runtime Java Path for OSX in Gradle (#49245) (#49246) + + On OSX the `bin` directory is nested under `Contents/Home` + relative to where it is for the other platforms. + +commit 288217e82bfd1267107afdd5b5b082a0b9787f92 +Author: Antoine Garcia +Date: Mon Nov 18 16:37:51 2019 +0100 + + [Docs] Specify field types not supporting doc values (#49041) + + The `string` type (with option `analyzed`) has been replaced by `text` after `6.0`, + also the `annonated_text` field do not support doc values and should be mentioned. + +commit 5f9965e4b83d447add02612015c46b7775156ec8 +Author: Przemysław Witek +Date: Mon Nov 18 14:58:20 2019 +0100 + + Lower minimum model memory limit value from 1MB to 1kB. (#49227) (#49242) + +commit f7d9e7bdc48e04e66f6d71de0929ab79c56efa2e +Author: Armin Braun +Date: Mon Nov 18 14:12:55 2019 +0100 + + Better Exceptions on Concurrent Snapshot Operations (#49220) (#49237) + + * Better Exceptions on Concurrent Snapshot Operations + + It is somewhat tricky to debug test failures from concurrent operations + without having the exact knowledge of what ran concurrently so I added + it to these exceptions in all spots. + +commit a817cf5e5c59b2d8ea3eb1632f3b722f4be9887b +Author: Armin Braun +Date: Mon Nov 18 13:02:20 2019 +0100 + + Fix testUnkownPlatform (#49235) + + * Adjust assertion to include `mac` to fix build tools tests + +commit 42268f0b0ea09c17ee0c79b456e919c404692443 +Author: Armin Braun +Date: Mon Nov 18 12:02:27 2019 +0100 + + Fix Broken Network Disruption in SnapshotResiliencyTests (#49216) (#49231) + + The network disruption was acting on node ids and node names + which made reconnects not work. Moved all usages to node names + to fix this. Since the map of all nodes in the test is indexed + by name this was easier to work with. + +commit af797a77a1a82f14ec03ff502032f557e40b0c94 +Author: Yannick Welsch +Date: Fri Nov 15 14:25:32 2019 +0100 + + Auto-expand indices according to allocation filtering rules (#48974) + + Honours allocation filtering rules when auto-expanding indices. + +commit ca4f55f2e40b82c484d5e4b4c0df77d10b21d395 +Author: Tanguy Leroux +Date: Mon Nov 18 05:56:59 2019 -0500 + + Add docker-compose fixtures for S3 integration tests (#49107) (#49229) + + Similarly to what has been done for Azure (#48636) and GCS (#48762), + this committ removes the existing Ant fixture that emulates a S3 storage + service in favor of multiple docker-compose based fixtures. + + The goals here are multiple: be able to reuse a s3-fixture outside of the + repository-s3 plugin; allow parallel execution of integration tests; removes + the existing AmazonS3Fixture that has evolved in a weird beast in + dedicated, more maintainable fixtures. + + The server side logic that emulates S3 mostly comes from the latest + HttpHandler made for S3 blob store repository tests, with additional + features extracted from the (now removed) AmazonS3Fixture: + authentication checks, session token checks and improved response + errors. Chunked upload request support for S3 object has been added + too. + + The server side logic of all tests now reside in a single S3HttpHandler class. + + Whereas AmazonS3Fixture contained logic for basic tests, session token + tests, EC2 tests or ECS tests, the S3 fixtures are now dedicated to each + kind of test. Fixtures are inheriting from each other, making things easier + to maintain. + +commit fcde1e752f321edd12390185b9bdf9effdebcff7 +Author: Armin Braun +Date: Mon Nov 18 11:22:28 2019 +0100 + + Fix Build on OSX + + OSX shows up as `mac` for the platform as well. + +commit ca912624ec638f5e266056f31d23cab16e2b0249 +Author: Hendrik Muhs +Date: Mon Nov 18 09:57:56 2019 +0100 + + [Transform] improve error handling of script errors (#48887) + + improve error handling for script errors, treating it as irrecoverable errors which puts the task + immediately into failed state, also improves the error extraction to properly report the script + error. + + fixes #48467 + +commit 23a26cede51a61ed0c858e9cee45dbc17df0ce1e +Author: Alpar Torok +Date: Mon Nov 18 11:13:40 2019 +0200 + + Fix java version of build-tools test + +commit fcac3fbfd9d4f06ca65183d78dc254d8ae4834f5 +Author: Tanguy Leroux +Date: Mon Nov 18 03:30:03 2019 -0500 + + AutoFollowIT should not rely on assertBusy but should use latches instead (#49141) + + AutoFollowIT relies on assertBusy() calls to wait for a given number of + leader indices to be created but this is prone to failures on CI. Instead, + we should use latches to indicate when auto-follow patterns must be + paused and resumed. + +commit 805c31e19e9d3e8576b1c2173986319dc90c21fc +Author: Dimitris Athanasiou +Date: Mon Nov 18 10:33:07 2019 +0200 + + [7.x][ML] Avoid NPE when node load is calculated on job assignment (#49186) (#49214) + + This commit fixes a NPE problem as reported in #49150. + But this problem uncovered that we never added proper handling + of state for data frame analytics tasks. + + In this commit we improve the `MlTasks.getDataFrameAnalyticsState` + method to handle null tasks and state tasks properly. + + Closes #49150 + + Backport of #49186 + +commit 6e775cfc970fe096c08e7cb822324a696a994e25 +Author: Alpar Torok +Date: Mon Nov 18 10:27:28 2019 +0200 + + Provision the correct JDK for test tasks (#48561) + + This PR adds build configuration to use the `jdk-download` plugin with + unit tests when no runtime java is configured externally. + + It's a first part in a longer chain of changes described in #40531. + +commit e84e21174bd10c4b0ac94057e5740f0a8bbff9a9 +Author: Rory Hunter +Date: Mon Nov 18 08:22:35 2019 +0000 + + Support `_FILE` suffixed env vars in Docker entrypoint (#49182) + + Backport of #47573. + + Closes #43603. Allow environment variables to be passed to ES in a Docker + container via a file, by setting an environment variable with the `_FILE` + suffix that points to the file with the intended value of the env var. + +commit b29f4dd9c2c4270ce76127b40cb36b4287588122 +Author: Rory Hunter +Date: Mon Nov 18 08:05:53 2019 +0000 + + Remove default indent from .editorconfig (#49183) + + Backport of #49136. + + Prior to 3a3e5f6, there was no default indent configured in + `.editorconfig`. This changed to 4 spaces when we configured an explicit + indent of 2 for Gradle files. However, this change meant that YAML files + then had 4-space indents, which is valid, but the repo's YAML files + typically use 2-space indents. + + Remove the default indent again, and instead explicitly set an indent size + for a variety of file types. + +commit 150db2b544051315b3d144f81788344ce3975ab2 +Author: Przemysław Witek +Date: Mon Nov 18 07:52:57 2019 +0100 + + Throw an exception when memory usage estimation endpoint encounters empty data frame. (#49143) (#49164) + +commit b9a571eb4337c21be31af6b16a28fcd5c248ac28 +Author: Jason Tedor +Date: Sun Nov 17 13:39:16 2019 -0500 + + Bundle JDK 13.0.1+9 with Windows + + This commit upgrades the JDK that is bundled with Windows from 13+33 to + 13.0.1+9. Our other platforms have previously been upgraded but Windows + was delayed because the artifacts were not available at the time that we + made the previous upgrade. + + Relates #48587 + +commit 60d1d67aac6bc598a9b05a8f87cbdb06377817ad +Author: Jason Tedor +Date: Sun Nov 17 12:47:44 2019 -0500 + + CCR should auto-retry rejected execution exceptions (#49213) + + If CCR encounters a rejected execution exception, today we treat this as + fatal. This is not though, as the stuffed queue could drain. Requiring + an administrator to manually restart the follow tasks that faced such an + exception is a burden. This commit addresses this by making CCR + auto-retry on rejected execution exceptions. + +commit 09a9ec4d2335ae5de5191ccc377d17d8a8be9989 +Author: Lisa Cawley +Date: Fri Nov 15 15:33:12 2019 -0800 + + [DOCS] Merges duplicate pages for native realms (#49198) + +commit 0e933a093d0a5aaab96a4853a472ec96d4d8443e +Author: Mayya Sharipova +Date: Fri Nov 15 16:01:37 2019 -0500 + + Add index name to search requests (#49175) + + We can't guarantee expected request failures if search request is across + many indexes, as if expected shards fail, some indexes may return 200. + + closes #47743 + +commit 2886d4c6ddb44ab29b51f9145358121a95f7a1fd +Author: Armin Braun +Date: Fri Nov 15 21:14:53 2019 +0100 + + Make FsBlobContainer Listing Resilient to Concurrent Modifications (#49142) (#49176) + + * Make FsBlobContainer Listing Resilient to Concurrent Modifications + + If we list out files in a folder via the lazily computed directory + stream, we have to deal with concurrent deletes when reading the file + attributes since we don't have a lock on the directory in any way. + + Closes #37581 + +commit 57f57227acafd00a63fb7241945801424bb7e80b +Author: Jay Modi +Date: Fri Nov 15 13:02:21 2019 -0700 + + Clean up static web server in sql-client tests (#49187) (#49197) + + The JdbcHttpClientRequestTests and HttpClientRequestTests classes both + hold a static reference to a mock web server that internally uses the + JDKs built-in HttpServer, which resides in a sun package that the + RamUsageEstimator does not have access to. This causes builds that use + a runtime of Java 8 to fail since the StaticFieldsInvariantRule is run + when Java 8 is used. + + Relates #41526 + Relates #49105 + +commit bc6a9de2dde812df43cf8ac5ba56b2967c8cd385 +Author: Lisa Cawley +Date: Fri Nov 15 10:50:15 2019 -0800 + + [DOCS] Edits the get tokens API (#45312) + +commit ec73e8c34c88613e470525fba79d2526fb338dfb +Author: Mark Vieira +Date: Fri Nov 15 09:56:38 2019 -0800 + + Add Elasticsearch 7.4.3 to BWC test matrix + +commit 69e69f5bffd5dfaf1f89d24707ec9978fd59a15e +Author: Mark Vieira +Date: Fri Nov 15 06:50:40 2019 -0800 + + Fix cases where we would unintentionally tar the entire build workspace (#49130) + + Fix cases where we would unintentionally tar the entire build workspace + +commit de8107e350f2d8ef96baf675e005b1e6aa5cf3d2 +Author: Lisa Cawley +Date: Fri Nov 15 09:36:39 2019 -0800 + + [DOCS] Adds ml-cpp PRs to release notes (#49185) + +commit dad68c59fee5a484aa6b349a1ea957204d22295e +Author: Mark Tozzi +Date: Fri Nov 15 12:32:26 2019 -0500 + + Avoid precision loss in DocValueFormat.RAW#parseLong (#49063) (#49169) + +commit aa4d86c5e9cd0290f20ed0936779eebc470e9501 +Author: William Brafford +Date: Fri Nov 15 11:53:28 2019 -0500 + + Remove parsed JVM settings from general settings (#49061) (#49170) + + The Apache Commons Daemon has some helpful features for Java + applications, like nice little next boxes for min heap, max heap, and + thread stack size. Our elasticsearch-service.bat script parses those + values out of the ES_JAVA_OPTS environment variable and provides them to + the Apache Commons Daemon invocation command in order to provide + sensible defaults. However, we failed to remove those values from the + ES_JAVA_OPTS environment variable, which meant they ended up in the + "Java Options" text box and would, from there, override whatever the + user put in the specific boxes for heap size or thread stack size. + + This commit modifies the loop that parses ES_JAVA_OPTS to construct a + new enviroment variable containing only the values that aren't parsed + out for heap size or thread stack size, then uses that new enviroment + variable in the commons daemon invocation command. + +commit c3745b03ee032d77290471e391a1336d038b7f66 +Author: markharwood +Date: Fri Nov 15 16:50:32 2019 +0000 + + Search optimisation - add canMatch early aborts for queries on "_index" field (#49158) + + Make queries on the “_index” field fast-fail if the target shard is an index that doesn’t match the query expression. Part of the “canMatch” phase optimisations. + + Closes #48473 + +commit b6ec066ca92741f8efad0786c5eb661d527529f9 +Author: Jay Modi +Date: Fri Nov 15 09:39:57 2019 -0700 + + ESIntegTestCase always cleans up static fields (#49105) (#49108) + + ESIntegTestCase has logic to clean up static fields in a method + annotated with `@AfterClass` so that these fields do not trigger the + StaticFieldsInvariantRule. However, during the exceptional close of the + test cluster, this cleanup can be missed. The StaticFieldsInvariantRule + always runs and will attempt to inspect the size of the static fields + that were not cleaned up. If the `currentCluster` field of + ESIntegTestCase references an InternalTestCluster, this could hold a + reference to an implementation of a `Path` that comes from the + `sun.nio.fs` package, which the security manager will deny access to. + This casues additional noise to be generated since the + AccessControlException will cause the StaticFieldsInvariantRule to fail + and also be reported along with the actual exception that occurred. + + This change clears the static fields of ESIntegTestCase in a finally + block inside the `@AfterClass` method to prevent this unnecessary noise. + + Closes #41526 + +commit eca93fcc5f9b2edd20eb60771c34920bbcb11c17 +Author: Lisa Cawley +Date: Fri Nov 15 11:27:36 2019 -0500 + + [DOCS] Adds machine learning node type and filters (#49121) + +commit 680436dd0d4938487df4bbafcea3a0b96b109bf3 +Author: Lee Hinman +Date: Fri Nov 15 09:22:37 2019 -0700 + + [7.x] Don't halt policy execution on policy trigger exception… (#49171) + + When triggered either by becoming master, a new cluster state, or a + periodic schedule, an ILM policy execution through + `maybeRunAsyncAction`, `runPolicyAfterStateChange`, or + `runPeriodicStep` throwing an exception will cause the loop the + terminate. This means that any indices that would have been processed + after the index where the exception was thrown will not be processed by + ILM. + + For most execution this is not a problem because the actual running of + steps is protected by a try/catch that moves the index to the ERROR step + in the event of a problem. If an exception occurs prior to step + execution (for example, in fetching and parsing the current + policy/step) however, it causes the loop termination previously + mentioned. + + This commit wraps the invocation of the methods specified above in a + try/catch block that provides better logging and does not bubble the + exception up. + +commit 36dc54481944bbb81fa7d2667853f9d9a6f67240 +Author: Jason Tedor +Date: Fri Nov 15 09:34:35 2019 -0500 + + Adjust version on ingest processor exception + + The dedicated ingest processor exception was backported to 7.5. This + commit updates the version in the 7.x branch. + +commit 32c3416549e3ff28d478022c5b4bc33b478750a4 +Author: Alpar Torok +Date: Fri Nov 15 16:25:28 2019 +0200 + + Move env vars to script, add build scan wrapper (#49070) + + * Move env vars to script, add build scan wrapper + + Having the env vars being set up in the build script makes it easier to + reason about. + This PR also adds the gradle scan wrapper back as support for it was + added t o jjbb. + + * Add explanatory comment + +commit 6075c30c6c026c95f6701f9e6c0069e8c13994ce +Author: Jason Tedor +Date: Fri Nov 15 08:51:29 2019 -0500 + + Restrict support for CMS to pre-JDK 14 (#49123) + + JDK 14 has removed CMS. This commit restricts the support for CMS to JDK + 8 through JDK 13, and defaults to G1 GC on JDK 14. We will revisit all + defaults in the future, but this ensures that we run with a + properly-configured garbage collector on JDK 14+. + +commit a000d868a0dc9882db148ba9a4cd53fe56edc55a +Author: Dimitris Athanasiou +Date: Fri Nov 15 15:56:53 2019 +0200 + + [7.x] Also expect RELOCATING in RecoveryIT.testRecoveryClosedIndex (#49090) (#49154) + + In addition to the fix in #48506, it seems we could also get the + `RELOCATING` state. + + This fixes the failure in + https://gradle-enterprise.elastic.co/s/svjmmvqk32cii/tests/ua7icdau7nz6y-2cwvhj3qg5qou?openStackTraces=WzBd + +commit 89b3c32b40767632ac81477e817d9048489bce71 +Author: Albert Zaharovits +Date: Fri Nov 15 08:44:09 2019 -0500 + + Audit log filter and marker (#49145) + + This adds a log marker and a marker filter for the audit log. + + Closes #47251 + +commit d9f0245b108806fb9265250ff07fad8e04e272db +Author: Christos Soulios <1561376+csoulios@users.noreply.github.com> +Date: Fri Nov 15 14:36:21 2019 +0200 + + [7.x] Implement stats aggregation for string terms (#49097) + + Backport of #47468 to 7.x + + This PR adds a new metric aggregation called string_stats that operates on string terms of a document and returns the following: + + min_length: The length of the shortest term + max_length: The length of the longest term + avg_length: The average length of all terms + distribution: The probability distribution of all characters appearing in all terms + entropy: The total Shannon entropy value calculated for all terms + + This aggregation has been implemented as an analytics plugin. + +commit 085d08cfd19a4d535ebf6dc3728825c3aa037e7a +Author: Andrei Dan +Date: Fri Nov 15 12:06:22 2019 +0000 + + ILM Remove obsolete testRolloverAlreadyExists (#49104) (#49144) + + The rollover action is now a retryable step (see #48256) + so ILM will keep retrying until it succeeds as opposed to stopping and + moving the execution in the ERROR step. + + Fixes #49073 + + (cherry picked from commit 3ae90898121b43032ec8f3b50514d93a86e14d0f) + Signed-off-by: Andrei Dan + + # Conflicts: + # x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java + +commit f5f0e1366ad3f50630f7b0dae84df3088de6b716 +Author: Ioannis Kakavas +Date: Fri Nov 15 11:54:08 2019 +0200 + + Handle unexpected/unchecked exceptions correctly (#49080) (#49137) + + Ensures that methods that are called from different threads ( i.e. + from the callbacks of org.apache.http.concurrent.FutureCallback ) + catch `Exception` instead of only the expected checked exceptions. + + This resolves a bug where OpenIdConnectAuthenticator#mergeObjects + would throw an IllegalStateException that was never caught causing + the thread to hang and the listener to never be called. This would + in turn cause Kibana requests to authenticate with OpenID Connect + to timeout and fail without even logging anything relevant. + + This also guards against unexpected Exceptions that might be thrown + by invoked library methods while performing the necessary operations + in these callbacks. + +commit fc505aaa7646eb28c9c14b0e58d4a24be5edb57b +Author: Armin Braun +Date: Fri Nov 15 09:54:53 2019 +0100 + + Track Repository Gen. in BlobStoreRepository (#48944) (#49116) + + This is intended as a stop-gap solution/improvement to #38941 that + prevents repo modifications without an intermittent master failover + from causing inconsistent (outdated due to inconsistent listing of index-N blobs) + `RepositoryData` to be written. + + Tracking the latest repository generation will move to the cluster state in a + separate pull request. This is intended as a low-risk change to be backported as + far as possible and motived by the recently increased chance of #38941 + causing trouble via SLM (see https://github.com/elastic/elasticsearch/issues/47520). + + Closes #47834 + Closes #49048 + +commit a370008856084f34b360c142e590319250bce39f +Author: Mark Vieira +Date: Thu Nov 14 16:44:41 2019 -0800 + + Disable thirdPartyAudit tests when running in a FIPS JVM (#49115) + + This fixes a regression introduced in #42042. The logic here was + mistakenly inverted such that we only run these tests in a FIPS JVM + which is the opposite of what we intend. + +commit 6f14a63059880ff8c912b3b9624f1d6baa4d9a29 +Author: Jason Tedor +Date: Thu Nov 14 18:40:07 2019 -0500 + + Remove the OSS UBI builds (#49118) + + This commit removes the OSS UBI-based Docker images from the build as we + are not going to publish these images for the time being. + +commit 961ffaf0ff06cda4dec575555dca71649efa3fd8 +Author: Ryan Ernst +Date: Thu Nov 14 16:27:59 2019 -0600 + + Omit docker from bats distributions (#49062) + + The bats tests require several distributions to all be built into a + single directory. The addition of docker packaging tests now cause the + bats tests to depend on docker, even though docker is not used there. + This commit filters out docker distributions from those that bats + depends on. + +commit 5cd6f64f1578a497314a43c83c73193320e9ff3f +Author: Tal Levy +Date: Thu Nov 14 14:15:34 2019 -0800 + + Introduce faster approximate sinh/atan math functions (#49009) (#49110) + + This commit introduces a new class called ESSloppyMath + that is meant to reflect the purpose of Lucene's SloppyMath, + but add additional unimplemented faster alternatives to math functions. + + The two that are used by geotile-grid a lot are sinh/atan. + + In a quick elasticsearch rally benchmark for geotile-grid on Switzerland + data points, this shows a (1.22x) 22% speed-up over using Math's functions. + + closes #41166. + +commit 085d9c6e82cf4e83fe03e419d2e6339dabdcacfc +Author: Jay Modi +Date: Thu Nov 14 13:05:47 2019 -0700 + + Reduce CPU usage of gradle run (#49055) (#49102) + + The RunTask is responsible for logging output from nodes to the console + and also stays active since we want the cluster to keep running. + However, the implementation of the logging and waiting resulted in a + spin loop that continually polls for data to have been written to one + of the nodes' output files. On my laptop, this causes an idle + invocation of `gradle run` to consume an entire core. + + The JDK provides a method to be notified of changes to files through + the use of a WatchService. While a WatchService based implementation + for logging and waiting works, a delay of up to ten seconds is + encountered when running on macOS. This is due to the lack of a native + WatchService implementation that uses kqueue or FSEvents; the current + WatchService implementation in the JDK uses polling with a default + interval of ten seconds. While the interval can be changed + programmatically it is not an acceptable solution due to the need to + access the com.sun.nio.file.SensitivityWatchEventModifier enum, which + is in an internal package. + + The change in this commit instead introduces a check to see if any data + was available to read and log. If no data is available in any of the + node output files, the thread sleeps for 100ms. This is enough time to + prevent consuming large amounts of cpu while still providing output to + the console in a timely fashion. + +commit 6bb6adb8d3dd042f3776f3d68646d33c0499043c +Author: James Baiera +Date: Thu Nov 14 14:14:39 2019 -0500 + + Reuse collected cluster state in EnrichPolicyRunner (#48488) (#49100) + + The cluster state is obtained twice in the EnrichPolicyRunner when updating + the final alias. There is a possibility for the state to be slightly different + between those two calls. This PR just has the function get the cluster state + once and reuse it for the life of the function call. + +commit 6ce04429c6068fe1c8398aa93c1e4b12e8b165e2 +Author: bellengao +Date: Fri Nov 15 02:50:46 2019 +0800 + + Fix `_analyze` API to correctly use normalizers when specified (#48866) + + Currently the `_analyze` endpoint doesn't correctly use normalizers specified + in the request. This change fixes that by returning the resolved normalizer from + TransportAnalyzeAction#getAnalyzer and updates test to be able to catch this + in the future. + + Closes #48650 + +commit e8f49cdee06e1faf4c8ba95269c6384ac2ff333d +Author: SylvainJuge +Date: Thu Nov 14 15:56:13 2019 +0100 + + [DOCS] minor fix to documentation: http.host can't default to itself (#48135) + + fix minor typos on http.host and transport.host default values. + + 7.x backport of https://github.com/elastic/elasticsearch/pull/48135 + +commit 2bcdcb17cdd19ecd0336c8760a93243e4e6630e5 +Author: Jason Tedor +Date: Thu Nov 14 09:53:08 2019 -0600 + + Introduce dedicated ingest processor exception (#48810) + + Today we wrap exceptions that occur while executing an ingest processor + in an ElasticsearchException. Today, in ExceptionsHelper#unwrapCause we + only unwrap causes for exceptions that implement + ElasticsearchWrapperException, which the top-level + ElasticsearchException does not. Ultimately, this means that any + exception that occurs during processor execution does not have its cause + unwrapped, and so its status is blanket treated as a 500. This means + that while executing a bulk request with an ingest pipeline, + document-level failures that occur during a processor will cause the + status for that document to be treated as 500. Since that does not give + the client any indication that they made a mistake, it means some + clients will enter infinite retries, thinking that there is some + server-side problem that merely needs to clear. This commit addresses + this by introducing a dedicated ingest processor exception, so that its + causes can be unwrapped. While we could consider a broader change to + unwrap causes for more than just ElasticsearchWrapperExceptions, that is + a broad change with unclear implications. Since the problem of reporting + 500s on client errors is a user-facing bug, we take the conservative + approach for now, and we can revisit the unwrapping in a future change. + +commit cac9fe4d866db93d390f63902475b81a25337644 +Author: Dan Hermann +Date: Thu Nov 14 09:39:28 2019 -0600 + + [7.x] Validate monitoring password at parse time (#49083) + +commit 6c5644335f062d81a59b562ad3d263fbf3ea7cca +Author: Christoph Büscher +Date: Thu Nov 14 14:47:55 2019 +0100 + + Simplify TransportMultiSearchActionTests (#48523) + + The test doesn't seem to need the threadpool that is created and destroyed in + setup and teardown any longer, so it can be removed. + +commit be5894ed9c77fc8072308480ecd57f751f425cb4 +Author: Dimitris Athanasiou +Date: Thu Nov 14 15:15:55 2019 +0200 + + [7.x][SQL] Mute JdbcConfigurationTests.testDriverConfigurationWithSSLInURL (#49085) (#49086) + + Relates #41557 + +commit e1726fff5681ab4f98f4f26e703bc9ea5ebacbc9 +Author: James Rodewig +Date: Thu Nov 14 07:59:56 2019 -0500 + + [DOCS] Reformat update license API docs (#48967) + + Makes a few changes to better align the update license API docs with + the [API reference template][0]. + + Changes: + + * Replaces POST with PUT in several snippet examples. + While both are valid, PUT is a bit more RESTful. + + * Removes leading slashes (/) from all snippets. + + * Relocates and retitles the 'Authorization' section to 'Prerequisites'. + + * Replaces explicit titles with the appropriate API reference template + attributes. + + * Replaces unneeded `[float]` tags with explicit anchors. + + Closes #35341 + + [0]: https://github.com/elastic/docs/blob/master/shared/api-ref-ex.asciidoc + +commit c46a0e87089f8c4be301ac729df569af3b9ab836 +Author: Rory Hunter +Date: Thu Nov 14 11:01:23 2019 +0000 + + Apply 2-space indent to all gradle scripts (#49071) + + Backport of #48849. Update `.editorconfig` to make the Java settings the + default for all files, and then apply a 2-space indent to all `*.gradle` + files. Then reformat all the files. + +commit 7c3198ba448038aabeb93d52e5070ddedb936deb +Author: Marios Trivyzas +Date: Thu Nov 14 11:29:57 2019 +0100 + + SQL: [Tests] Mute testReplaceChildren for Pivot (#49045) + + Temporarily "mute" the testReplaceChildren for Pivot since it leads to + failing tests for some seeds, since the new child doesn't respond to a + valid data type. + + Relates to #48900 + + (cherry picked from commit 6200a2207b9a4264d2f3fc976577323c7e084317) + +commit 25e05b0013caf359cac62b2ca53ae3e48d3ed70c +Author: Armin Braun +Date: Wed Nov 13 22:06:55 2019 +0100 + + Fix X-Pack SchedulerEngine Shutdown (#48951) (#49054) + + We can have a race here where `scheduleNextRun` executes concurrently to `stop` + and so we run into a `RejectedExecutionException` that we don't catch and thus it + fails tests. + => Fixed by ignoring these so long as they coincide with a scheduler shutdown + +commit 095c34359fa091259c3ccfc1e18023b3e7ae7edb +Author: James Rodewig +Date: Wed Nov 13 14:27:10 2019 -0500 + + [DOCS] Note limitations of `max_gram` parm in `edge_ngram` tokenizer for index analyzers (#49007) + + The `edge_ngram` tokenizer limits tokens to the `max_gram` character + length. Autocomplete searches for terms longer than this limit return + no results. + + To prevent this, you can use the `truncate` token filter to truncate + tokens to the `max_gram` character length. However, this could return irrelevant results. + + This commit adds some advisory text to make users aware of this limitation and outline the tradeoffs for each approach. + + Closes #48956. + +commit e6ad3c29fd2b5d74fa2178159202c2479ac502d9 +Author: Przemysław Witek +Date: Wed Nov 13 20:23:13 2019 +0100 + + Do not throw exceptions resulting from persisting datafeed timing stats. (#49044) (#49050) + +commit 66f0c8900ff4f80f4f12dd2b1c29c75ac35a1f06 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Nov 13 18:48:05 2019 +0100 + + Fix Transport Stopped Exception (#48930) (#49035) + + When a node shuts down, `TransportService` moves to stopped state and + then closes connections. If a request is done in between, an exception + was thrown that was not retried in replication actions. Now throw a + wrapped `NodeClosedException` exception instead, which is correctly + handled in replication action. Fixed other usages too. + + Relates #42612 + +commit fb685adc9486cad52166bb0d9201423b412a41a3 +Author: Alpar Torok +Date: Wed Nov 13 17:12:00 2019 +0200 + + Move periodic job to ES repo (#48570) + + * Move periodic job to ES repo + + This change kickstarts the process of moving CI job definitions to this + repo. + + * Added a minimal readme to provide pointers to the documentation + + * Update .ci/README.md + + Co-Authored-By: Rory Hunter + + * Update .ci/README.md + + Co-Authored-By: Rory Hunter + + * point to main repo + + * PR review + + * Add link to JJBB + +commit 20fc1dbe18f839b1e85b37aec39524dcd55c7c4b +Author: Tanguy Leroux +Date: Wed Nov 13 10:03:59 2019 -0500 + + Move MinIO fixture in its own project (#49036) + + This commit moves the MinIO docker-compose fixture from the + :plugins:repository-s3 to its own :test:minio-fixture Gradle project. + +commit 838af15d295d3f3e42b9527e8053e357187d8fa9 +Author: James Rodewig +Date: Wed Nov 13 09:35:00 2019 -0500 + + [DOCS] Reformat compound word token filters (#49006) + + * Separates the compound token filters doc pages into separate token + filter pages: + * Dictionary decompounder token filter + * Hyphenation decompounder token filter + + * Adds analyze API examples for each compound token filter + + * Adds a redirect for the removed compound token filters page + + Co-Authored-By: debadair + +commit b55022b59f7573cd6150641d3a09be0b30321fb6 +Author: István Zoltán Szabó +Date: Wed Nov 13 10:21:43 2019 +0100 + + [DOCS] Adds test clause to the code snippets in the cluster restart page (#49023) + +commit dc50d3fc461c4b62eb2d215b7e50157bedb51c69 +Author: Christoph Büscher +Date: Wed Nov 13 14:33:10 2019 +0100 + + Add negative boolean expression note to CONTRIBUTING.md (#49033) + + The team sometimes get questions around the use of `!foo` vs. `foo == false` in + PRs and reviews (e.g. #48615). This change adds a bullet point to CONTRIBUTING.md + to make expectations here clearer and gives us something to point to in case of + discussion. + +commit e86b598813f364c0fb13ea9a9fe6208d36001ffb +Author: Tanguy Leroux +Date: Wed Nov 13 07:17:26 2019 -0500 + + Fix AutoFollowIT (#49025) + + This commit fixes an off-by-one bug in the AutoFollowIT test that causes + failures because the leaderIndices counter is incremented during the evaluation + of the leaderIndices.incrementAndGet() < 20 condition but the 20th index is + not created, making the final assertion not verified. + + It also gives a bit more time for cluster state updates to be processed on the + follower cluster. + + Closes #48982 + +commit f0e386d60dab842cc690a75ac819071fee2572e0 +Author: Alan Woodward +Date: Wed Nov 13 11:49:49 2019 +0000 + + Fix ResizeResponseTests randomization + +commit 44050429005e5b5ad399fa276826a77a4574b820 +Author: Ioannis Kakavas +Date: Wed Nov 13 13:43:56 2019 +0200 + + Remove unnecessary details logged for OIDC (#48746) (#49031) + + This commit removes unnecessary details logged for + OIDC. + + Co-Authored-By: Ioannis Kakavas + +commit 2dfa0133d5f445874d91e80be8c1405493f41be3 +Author: Yannick Welsch +Date: Wed Nov 13 11:42:13 2019 +0100 + + Always use primary term from primary to index docs on replica (#47583) + + Ensures that we always use the primary term established by the primary to index docs on the + replica. Makes the logic around replication less brittle by always using the operation primary + term on the replica that is coming from the primary. + +commit 999d66fc87d82a95fc962edcbea0df89ab36241b +Author: Alan Woodward +Date: Wed Nov 13 10:23:53 2019 +0000 + + Add client-side ResizeRequest and ResizeResponse classes (#48937) + + Closes #48468 + +commit e0331e2a0f087a4d0fad3df0b71b4b2e4be02a41 +Author: Ioannis Kakavas +Date: Wed Nov 13 12:10:01 2019 +0200 + + Remove limitation for SAML encryption in FIPS mode (#48948) (#49019) + + Our documentation regarding FIPS 140 claimed that when using SAML + in a JVM that is configured in FIPS approved only mode, one could + not use encrypted assertions. This stemmed from a wrong + understanding regarding the compliance of RSA-OAEP which is used + as the key wrapping algorithm for encrypting the key with which the + SAML Assertion is encrypted. + + However, as stated for instance in + https://downloads.bouncycastle.org/fips-java/BC-FJA-SecurityPolicy-1.0.0.pdf + RSA-OAEP is approved for key transport, so this limitation is not + effective. + + This change removes the limitation from our FIPS 140 related + documentation. + +commit 37fa3fb4ff6874b6dd9ddc33b1f5ffb4a02d46a3 +Author: Julie Tibshirani +Date: Tue Nov 12 21:50:39 2019 -0500 + + Ensure parameters are updated when merging flattened mappings. (#48971) (#49014) + + This PR makes the following two fixes around updating flattened fields: + + * Make sure that the new value for ignore_above is immediately taken into + affect. Previously we recorded the new value but did not use it when parsing + documents. + * Allow depth_limit to be updated dynamically. It seems plausible that a user + might want to tweak this setting as they encounter more data. + +commit 5eb37c29fec148eb4afe16c851857aa75bd793f9 +Author: Lee Hinman +Date: Tue Nov 12 19:41:06 2019 -0700 + + [7.x] Re-read policy phase JSON when using ILM's move-to-step… (#49011) + + When using the move-to-step API, we should reread the phase JSON from + the latest version of the ILM policy. This allows a user to move to the + same step while re-reading the policy's latest version. For example, + when changing rollover criteria. + + While manually messing around with some other things I discovered that + we only reread the policy when using the retry API, not the move-to-step + API. This commit changes the move-to-step API to always read the latest + version of the policy. + +commit 40776eedafca745ba8a187dead5e53aeb32a5ee5 +Author: Igor Motov +Date: Tue Nov 12 15:11:43 2019 -0500 + + Fix ignoring missing values in min/max aggregations (#48970) + + Fixes the issue when the missing values can be ignored in min/max + due to BKD optimization. + + Fixes #48905 + +commit 0e1035241da241e2c8611fa03ffe982142e8a47f +Author: Armin Braun +Date: Tue Nov 12 21:35:41 2019 +0100 + + Fix Broken Snapshots in Mixed Clusters (#48993) (#48995) + + Reverts #48947 and fixes the issue orginally addressed by removing the assertion. + It turns out we can't simply pass empty shard generations to the snapshot finalization in the + BwC case as that results in no indices being added to the meta for the given snapshot since + we take the indices from the shard generations (even in the BwC case the `null` generations work + fine for this). + + Closes #48983 + +commit 1903505a3f8d871295b5b2e4e3b7bb3c745e5341 +Author: Tanguy Leroux +Date: Tue Nov 12 14:13:36 2019 -0500 + + Log exceptions thrown by HttpHandlers in repository integration tests (#48991) + + This commit changes the ESMockAPIBasedRepositoryIntegTestCase so + that HttpHandler are now wrapped in order to log any exceptions that + could be thrown when executing the server side logic in repository + integration tests. + +commit 698ebd3d0a2eb375e1e66c106ad233e3979ecf9d +Author: David Roberts +Date: Tue Nov 12 17:31:07 2019 +0000 + + [TEST] Mute docs snippet test in close-job.asciidoc (#49000) + + Due to https://github.com/elastic/elasticsearch/pull/48583#issuecomment-552991325 + +commit bc23bc51464f3808e28f1a421a7251738dd6d914 +Author: Michael Basnight +Date: Mon Nov 11 16:47:22 2019 -0500 + + Add delete alias to the HLRC (#48819) + + The delete alias call is a rest only API call, but should still be added + to the rest client. This commit adds it as well as relevant tests. + + Ref #47678 + +commit 561351d2fcf98349c260a8081b79dd1e67bf7580 +Author: Orhan Toy +Date: Tue Nov 12 15:44:57 2019 +0100 + + [Docs] Fix _count HTTP method (#48979) + +commit fc145575c48e4c6146a55cea8be13f34257c1a0a +Author: István Zoltán Szabó +Date: Tue Nov 12 14:49:11 2019 +0100 + + [DOCS] Creates a cluster restart documentation page (#48583) + + Co-Authored-By: James Rodewig + +commit 42e92616f69ff0943ebbc4c0729408cdbafc8a7f +Author: James Rodewig +Date: Tue Nov 12 08:34:51 2019 -0500 + + [DOCS] Document indices response parameters for node stats API (#47525) + +commit 18d5d733056f2bc8b9a48a0e1fc1c145af1a5c69 +Author: Martijn van Groningen +Date: Tue Nov 12 13:22:34 2019 +0100 + + Enable spotless for enrich gradle project in 7 dot x branch. (#48976) + + Backport of #48908 + + The enrich project doesn't have much history as all the other gradle projects, + so it makes sense to enable spotless for this gradle project. + +commit 9baea808532bb851055fbb6a61ec4407fb874140 +Author: David Turner +Date: Tue Nov 12 11:02:18 2019 +0000 + + Ignore metadata of deleted indices at start (#48918) + + Today in 6.x it is possible to add an index tombstone to the graveyard without + deleting the corresponding index metadata, because the deletion is slightly + deferred. If you shut down the node and upgrade to 7.x when in this state then + the node will fail to apply any cluster states, reporting + + java.lang.IllegalStateException: Cannot delete index [...], it is still part of the cluster state. + + This commit addresses this situation by skipping over any index metadata with a + corresponding tombstone, allowing this metadata to be cleaned up by the 7.x + node. + +commit dc441588b6ec8ec333ca5504155a33e495322b3f +Author: David Turner +Date: Wed Nov 6 00:26:37 2019 +0100 + + Remove support for ancient corrupted markers (#48858) + + Today we still support reading store corruption markers of versions that + haven't been written since 1.7. This commit removes this legacy support. + +commit ab15bce4e792832eaacec75001fc6770fa172798 +Author: Yannick Welsch +Date: Tue Nov 12 11:57:06 2019 +0100 + + Auto-expand replicated closed indices (#48973) + + Fixes a bug where replicated closed indices were not being auto-expanded. + +commit 0e82b5f59be7ad85d9b9c1f28b9d6bb75ca2c4b4 +Author: jimczi +Date: Tue Nov 12 09:57:58 2019 +0100 + + add release notes for 7.5.0 + +commit 2906702901f3153cc764bcec741426980a68b564 +Author: Alpar Torok +Date: Tue Nov 12 10:08:07 2019 +0200 + + Always increase timeout for packaging tests (#48945) + + The previous approach did not work because the system property is passed + to Gradle but not to the tests JVM. + We shouldn't really pass this to the tests as we wouldn't want to have + differences. + This timeout being different might not be bad, but having a way to + differentiate could lead to others and it's best avoided. + +commit 5c6f8bab8df2bffddaa9935a891ed36d5f61f8f7 +Author: Alpar Torok +Date: Tue Nov 12 10:08:01 2019 +0200 + + Create a thread dump if ES fails to start in packaging tests (#48932) + + Relates to #15385. + Hoping to get some insight on CI failures. + +commit ea9f094e75433e75762483ea5ad1d757a6d7fea2 +Author: Armin Braun +Date: Tue Nov 12 08:49:40 2019 +0100 + + Significantly Lower Monitoring HttpExport Memory Footprint (#48854) (#48966) + + The `HttpExportBulk` exporter is using a lot more memory than it needs to + by allocating buffers for serialization and IO: + + * Remove copying of all bytes when flushing, instead use the stream wrapper + * Remove copying step turning the BAOS into a `byte[]` + * This also avoids the allocation of a single huge `byte[]` and instead makes use of the internal paging logic of the `BytesStreamOutput` + * Don't allocate a new BAOS for every document, just keep appending to a single BAOS + +commit 0645ee88e2a3fff562a055fba2eaf928653c0db3 +Author: Tim Brooks +Date: Mon Nov 11 18:42:02 2019 -0500 + + Send cluster name and discovery node in handshake (#48916) + + This commits sends the cluster name and discovery naode in the transport + level handshake response. This will allow us to stop sending the + transport service level handshake request in the 8.0-8.x release cycle. + It is necessary to start sending this in 7.x so that 8.0 is guaranteed + to be communicating with a version that sends the required information. + +commit c320b499a0b886faf718fcba805553b6e606f522 +Author: Jake Landis +Date: Mon Nov 11 16:31:21 2019 -0600 + + Prevent deadlock by using separate schedulers (#48697) (#48964) + + Currently the BulkProcessor class uses a single scheduler to schedule + flushes and retries. Functionally these are very different concerns but + can result in a dead lock. Specifically, the single shared scheduler + can kick off a flush task, which only finishes it's task when the bulk + that is being flushed finishes. If (for what ever reason), any items in + that bulk fails it will (by default) schedule a retry. However, that retry + will never run it's task, since the flush task is consuming the 1 and + only thread available from the shared scheduler. + + Since the BulkProcessor is mostly client based code, the client can + provide their own scheduler. As-is the scheduler would require + at minimum 2 worker threads to avoid the potential deadlock. Since the + number of threads is a configuration option in the scheduler, the code + can not enforce this 2 worker rule until runtime. For this reason this + commit splits the single task scheduler into 2 schedulers. This eliminates + the potential for the flush task to block the retry task and removes this + deadlock scenario. + + This commit also deprecates the Java APIs that presume a single scheduler, + and updates any internal code to no longer use those APIs. + + Fixes #47599 + + Note - #41451 fixed the general case where a bulk fails and is retried + that can result in a deadlock. This fix should address that case as well as + the case when a bulk failure *from the flush* needs to be retried. + +commit acae07113f27c49bd5ed2df2403833772e780a34 +Author: Jason Tedor +Date: Mon Nov 11 15:41:04 2019 -0500 + + Fix names of UBI-based Docker build contexts + + This commit fixes the names of the UBI-based Docker build contexts to + lift the ubi component of the name into the archive base name, instead + of the classifier. + +commit 46ab1db54f241680623468ade652e51f117d1992 +Author: Benjamin Trent +Date: Mon Nov 11 15:43:03 2019 -0500 + + [7.x] [ML] Add new geo_results.(actual_point|typical_point) fields for `lat_long` results (#47050) (#48958) + + * [ML] Add new geo_results.(actual_point|typical_point) fields for `lat_long` results (#47050) + + [ML] Add new geo_results.(actual_point|typical_point) fields for `lat_long` results (#47050) + + Related PR: https://github.com/elastic/ml-cpp/pull/809 + + * adjusting bwc version + +commit 8acbd0aa2a9c3d6cb06d0c42237e08a638ba4898 +Author: Mark Vieira +Date: Mon Nov 11 12:25:14 2019 -0800 + + Ensure client jar projects generate correct POM artifacts (#48961) + +commit d9e569278f57d54bfa1cf2b5787817c76ebbf6ab +Author: Mark Tozzi +Date: Mon Nov 11 15:09:19 2019 -0500 + + Refactor and DRY up Kahan Sum algorithm (#48558) (#48959) + +commit c45470f84f4f9024c1e7854b315f73e1170e5d75 +Author: Armin Braun +Date: Mon Nov 11 18:02:53 2019 +0100 + + Fix ShardGenerations in RepositoryData in BwC Case (#48920) (#48947) + + We were tripping the assertion that the makes sure we only have empty `ShardGenerations` in `RepositoryData` in the BwC case because shard generations were passed to the `Repository` in the BwC case. Fixed by only generating empty shard gen for BwC snapshots in `SnapshotsService`. + +commit 909fbd001561a1946d7bbcc64d53a7589e1f1319 +Author: Jake Landis +Date: Mon Nov 11 09:38:14 2019 -0600 + + [7.x] Mute FullClusterRestartTest#testWatcher and 30s timeout… (#48850) + + The timeout was increased to 60s to allow this test more time to reach a + yellow state. However, the test will still on occasion fail even with the + 60s timeout. + + Related: #48381 + Related: #48434 + Related: #47950 + Related: #40178 + +commit 6119f0aaa24e9faf538fe25d08630a6ad94c96eb +Author: Christoph Büscher +Date: Mon Nov 11 09:34:21 2019 -0500 + + Fix Eclipse compilation in DataFrameDataExtractorTests (#48942) + +commit a1dd830cb555f0a0a909c1a265006ba5f14fb665 +Author: Martijn van Groningen +Date: Thu Oct 31 14:11:25 2019 +0100 + + Re-enabled test with longer timeout waiting for monitoring. + + See #48258 + +commit c2f52015d3b2de2eff0e280800b93bde2c653bfb +Author: István Zoltán Szabó +Date: Mon Nov 11 10:00:11 2019 -0500 + + [DOCS] Removes best practice about fields that are highly correlated to the dependent variable. (#48935) + +commit 91888959e88cc6c826f043193db2656461a0e00c +Author: István Zoltán Szabó +Date: Mon Nov 11 09:53:59 2019 -0500 + + [DOCS] Extends analyzed_fields description in PUT DFA API docs. (#48307) + +commit 4b854986172d2013ccb94251eeedc07e26b8bab0 +Author: Patrick Maynard +Date: Mon Nov 11 15:38:18 2019 +0100 + + [DOCS] Fix typo in search type docs (#48868) + +commit 014e1b1090eb43faf47a246be7e1df32b0903493 +Author: Rory Hunter +Date: Mon Nov 11 14:33:04 2019 +0000 + + Improve resiliency to auto-formatting in server (#48940) + + Backport of #48450. + + Make a number of changes so that code in the `server` directory is more + resilient to automatic formatting. This covers: + + * Reformatting multiline JSON to embed whitespace in the strings + * Move some comments around to they aren't auto-formatted to a strange + place. This also required moving some `&&` and `||` operators from the + end-of-line to start-of-line`. + * Add helper method `reformatJson()`, to strip whitespace from a JSON + document using XContent methods. This is sometimes necessary where + a test is comparing some machine-generated JSON with an expected + value. + + Also, `HyperLogLogPlusPlus.java` is now excluded from formatting because it + contains large data tables that don't reformat well with the current settings, + and changing the settings would be worse for the rest of the codebase. + +commit dd92830801a225cf2a878279e3c655e76cbe372b +Author: James Rodewig +Date: Mon Nov 11 08:49:01 2019 -0500 + + [DOCS] Reformat condition token filter (#48775) + +commit eb0d8f33839ceae1aa5c8237d97d23e549298ce8 +Author: Rafael Acevedo +Date: Mon Nov 11 10:29:52 2019 -0300 + + update gradle to 5.6.4 (#48872) + +commit 35e21f85f3bb6f74a75919b9a0fe866f54629bf5 +Author: Rory Hunter +Date: Mon Nov 11 11:43:32 2019 +0000 + + Reenable Docker tests again (#48936) + + Backport of #48898. + + We no longer configure distributions for prior versions for Docker. This + is because doing so prompts Gradle to try and resolve the Docker + dependencies, which doesn't work as they can't be downloaded via Ivy + (configured in DistributionDownloadPlugin). Since we need these for the + BATS upgrade tests, and those tests only cover .rpm and .deb, it's OK to + omit creating such distributions in the first place. We may need to + revisit this in the future, to allow upgrade testing using Docker + containers. + +commit e33a1b794218de109cb0f0e60242ba1b39a53356 +Author: Alpar Torok +Date: Mon Nov 11 11:22:47 2019 +0200 + + Add links to infra-stats for scans generated in CI (#48732) + + * Add links to infra-stats for scans generated in CI + + It turns out we already gather system logs in infra-stats, and we have + system metrics too there. + + This PR adds a links to the logs we gather for the host the build is + runnig on. + And a link to the host overview in the infrastructure app tuned to 5 + minutes from before gradle started to 5 minutes after the scan was + generated. + + * add buildFinished + +commit f642baa9fbe7b59ac7469bf64dae0fe8aaf4af99 +Author: Arne Welzel +Date: Mon Nov 11 10:10:13 2019 +0100 + + [DOCS] Remove extra "when" (#48926) + +commit 87862868c6c648059ed5949385e8d130a3bc0554 +Author: Yannick Welsch +Date: Sat Nov 9 17:18:54 2019 +0100 + + Allow realtime get to read from translog (#48843) + + The realtime GET API currently has erratic performance in case where a document is accessed + that has just been indexed but not refreshed yet, as the implementation will currently force an + internal refresh in that case. Refreshing can be an expensive operation, and also will block the + thread that executes the GET operation, blocking other GETs to be processed. In case of + frequent access of recently indexed documents, this can lead to a refresh storm and terrible + GET performance. + + While older versions of Elasticsearch (2.x and older) did not trigger refreshes and instead opted + to read from the translog in case of realtime GET API or update API, this was removed in 5.0 + (#20102) to avoid inconsistencies between values that were returned from the translog and + those returned by the index. This was partially reverted in 6.3 (#29264) to allow _update and + upsert to read from the translog again as it was easier to guarantee consistency for these, and + also brought back more predictable performance characteristics of this API. Calls to the realtime + GET API, however, would still always do a refresh if necessary to return consistent results. This + means that users that were calling realtime GET APIs to coordinate updates on client side + (realtime GET + CAS for conditional index of updated doc) would still see very erratic + performance. + + This PR (together with #48707) resolves the inconsistencies between reading from translog and + index. In particular it fixes the inconsistencies that happen when requesting stored fields, which + were not available when reading from translog. In case where stored fields are requested, this + PR will reparse the _source from the translog and derive the stored fields to be returned. With + this, it changes the realtime GET API to allow reading from the translog again, avoid refresh + storms and blocking the GET threadpool, and provide overall much better and predictable + performance for this API. + +commit ff6c121eb944a2f08baa8b1e439a75686bb37629 +Author: Nhat Nguyen +Date: Thu Nov 7 15:39:46 2019 -0500 + + Closed shard should never open new engine (#47186) + + We should not open new engines if a shard is closed. We break this + assumption in #45263 where we stop verifying the shard state before + creating an engine but only before swapping the engine reference. + We can fail to snapshot the store metadata or checkIndex a closed shard + if there's some IndexWriter holding the index lock. + + Closes #47060 + +commit 9a42e71dd95a3154a1722572dcdd59a5e88fec13 +Author: Nhat Nguyen +Date: Fri Nov 1 09:23:46 2019 -0400 + + Do not cancel recovery for copy on broken node (#48265) + + This change fixes a poisonous situation where an ongoing recovery was + canceled because a better copy was found on a node that the cluster had + previously tried allocating the shard to but failed. The solution is to + keep track of the set of nodes that an allocation was failed on so that + we can avoid canceling the current recovery for a copy on failed nodes. + + Closes #47974 + +commit 5e4501eb3f5a34d96ab0fe0a3b462aa7f42de6de +Author: Julian Simioni +Date: Fri Nov 8 14:53:56 2019 -0500 + + [Docs] Consolidate single example into a single line (#48904) + + The first example of splitting rules for the `word_delimiter` token filter was spread across two bullet points. This makes it look like they are two separate splitting rules. + +commit af887be3e52ca7d538c6334dfc8d927b096d5769 +Author: Yannick Welsch +Date: Fri Nov 8 00:01:54 2019 -0500 + + Hide orphaned tasks from follower stats (#48901) + + CCR follower stats can return information for persistent tasks that are in the process of being cleaned up. This is problematic for tests where CCR follower indices have been deleted, but their persistent follower task is only cleaned up asynchronously afterwards. If one of the following tests then accesses the follower stats, it might still get the stats for that follower task. + + In addition, some tests were not cleaning up their auto-follow patterns, leaving orphaned patterns behind. Other tests cleaned up their auto-follow patterns. As always the same name was used, it just depended on the test execution order whether this led to a failure or not. This commit fixes the offensive tests, and will also automatically remove auto-follow-patterns at the end of tests, like we do for many other features. + + Closes #48700 + +commit 8835142ac9060038f2b7a3400b6d22e3ee3a66df +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Nov 8 00:04:00 2019 +0100 + + Grok processor ignore case test (#48909) + + Added test demonstrating that grok using ignore case works, since this + does a minimal test that the `joni` and `jcodings` libraries are + compatible. + + Forward-port of test from #43334 + +commit bdc7057d58fdf1185e363e70cf71bd97376df645 +Author: bellengao +Date: Fri Nov 8 04:26:55 2019 +0800 + + [DOCS] Correct typo in split index API docs (#48894) + +commit 8a14ea5567599273f0a55b56a604707b13b4d7d4 +Author: Tanguy Leroux +Date: Thu Nov 7 13:27:22 2019 -0500 + + Add docker-composed based test fixture for GCS (#48902) + + Similarly to what has be done for Azure in #48636, this commit + adds a new :test:fixtures:gcs-fixture project which provides two + docker-compose based fixtures that emulate a Google Cloud + Storage service. + + Some code has been extracted from existing tests and placed + into this new project so that it can be easily reused in other + projects. + +commit 293902c6a51020c88cb364a4480e1ddc5523853a +Author: bellengao +Date: Thu Nov 7 23:08:38 2019 +0800 + + [DOCS] Fix shard type in CCR overview doc (#48882) + + Closes #48875 + +commit df16ff777e547b6b93b97998c7d4825c9e3b9af7 +Author: Rory Hunter +Date: Thu Nov 7 14:28:33 2019 +0000 + + Disable docker packaging tests again (#48896) + + Backport of #48883. + + Per elastic/infra#15864, the Elasticsearch CI images are failing due to + a packer_cache failure. This is because Gradle is trying to resolve + a `.docker` file through the Ivy repository, which doesn't work. Disable + the Docker tests again until we figure out the way forward. + +commit 5805560a2a8eed2872fb5c89664b2641f7c599ce +Author: Dan Hermann +Date: Thu Nov 7 05:24:49 2019 -0600 + + Validate index name time format setting at parse time (#47911) (#48881) + +commit dfc6a13b442f204f3b5dba125e70a7011d0a89f8 +Author: Dimitris Athanasiou +Date: Thu Nov 7 07:30:50 2019 +0200 + + [7.x][ML] Handle nested arrays in source fields (#48885) (#48889) + + Backport of #48885 + +commit 3b9ce0a4f3bab85df2fe4a4911bd54a4eedec68b +Author: Adrien Grand +Date: Wed Nov 6 10:13:09 2019 -0500 + + Elasticsearch 7.5 is on Lucene 8.3. (#48831) + +commit 552381d7f94d263a891a50626e2c08d9cc79abc7 +Author: Tanguy Leroux +Date: Wed Nov 6 09:46:17 2019 -0500 + + Add mention to Pause Auto-Follower API in Upgrade Clusters docs (#48764) + + Relates #46665 + +commit 3c9bd13dca63626b22e0d98f6911c48d19600d6a +Author: István Zoltán Szabó +Date: Wed Nov 6 07:40:27 2019 -0500 + + [DOCS] Adds classification type DFA API docs and ml-shared.asciidoc (#48241) + +commit 70765dfb05a708e59bb5eefd306d31c2b3250b31 +Author: István Zoltán Szabó +Date: Wed Nov 6 07:37:14 2019 -0500 + + [DOCS] Adds classification type evaluation docs to the DFA evaluation API (#47657) + +commit f1396b63228af2b2f546b0baf109025c464aa6f3 +Author: James Rodewig +Date: Tue Nov 5 17:07:39 2019 -0500 + + [DOCS] Add Java to list of HTTP client libraries for basic authentication (#48647) + +commit bd5c6c47792c904c8bcaa51f26d39fdb3b8773a8 +Author: David Turner +Date: Tue Nov 5 18:08:22 2019 +0100 + + Add preflight check to dynamic mapping updates (#48867) + + Today if the primary discovers that an indexing request needs a mapping update + then it will send it to the master for validation and processing. If, however, + the put-mapping request is invalid then the master still processes it as a + (no-op) cluster state update. When there are a large number of indexing + operations that result in invalid mapping updates this can overwhelm the + master. + + However, the primary already has a reasonably up-to-date mapping against which + it can check the (approximate) validity of the put-mapping request before + sending it to the master. For instance it is not possible to remove fields in a + mapping update, so if the primary detects that a mapping update will exceed the + fields limit then it can reject it itself and avoid bothering the master. + + This commit adds a pre-flight check to the mapping update path so that the + primary can discard obviously-invalid put-mapping requests itself. + + Fixes #35564 + Backport of #48817 + +commit 24f7d4e83bd6dcc2e88fc4c1afaea2f83e181dd9 +Author: Rory Hunter +Date: Tue Nov 5 15:17:59 2019 +0000 + + Add Docker packaging tests on 7.x (#48857) + + Backport of #46599 and #47640. Add packaging tests for Docker. + + * Introduce packaging tests for Docker (#46599) + + Closes #37617. Add packaging tests for our Docker images, similar to what + we have for RPMs or Debian packages. This works by running a container and + probing it e.g. via `docker exec`. Test can also be run in Vagrant, by + exporting the Docker images to disk and loading them again in VMs. Docker + is installed via `Vagrantfile` in a selection of boxes. + + * Only define Docker pkg tests if Docker is available (#47640) + + Closes #47639, and unmutes tests that were muted in b958467. + + The Docker packaging tests were being defined irrespective of whether + Docker was actually available in the current environment. Instead, + implement exclude lists so that in environments where Docker is not + available, no Docker packaging tests are defined. For CI hosts, the build + checks `.ci/dockerOnLinuxExclusions`. The Vagrant VMs can defined the + extension property `shouldTestDocker` property to opt-in to packaging + tests. + + As part of this, define a seperate utility class for checking Docker, + and call that instead of defining checks in-line in BuildPlugin.groovy + +commit baabc21a049b8e9729f1def9ed28d8736665f785 +Author: glerb <56281590+glerb@users.noreply.github.com> +Date: Tue Nov 5 05:46:55 2019 -0800 + + [DOCS] Correct typo in Discovery docs (#48494) + +commit c03f7ba74cad0bf3253270a98b85dbc825b8494e +Author: David Roberts +Date: Tue Nov 5 06:47:46 2019 -0500 + + [TEST] Mute TimeoutCheckerTests.testWatchdog + + Due to https://github.com/elastic/elasticsearch/issues/48861 + +commit d83e3740622b2d2eed91b14c6185982344d1b327 +Author: Armin Braun +Date: Mon Nov 4 21:36:17 2019 +0100 + + Bound Linearizability Check in CoordinatorTests (#48751) (#48853) + + Same as #44444 but for the coordinator tests. + Closes #48742 + +commit c85cf7a6de3d59ad33dcb4873f617e18e696eb94 +Author: Dan Hermann +Date: Mon Nov 4 09:51:13 2019 -0600 + + Validate proxy base path at parse time (#47912) (#48825) + +commit 020ff0fef9e071cf98775000ba802d658794f2b2 +Author: Nhat Nguyen +Date: Sat Nov 2 18:44:38 2019 -0400 + + Do not intercept renew requests from other tests (#48833) + + We might have some outstanding renew retention lease requests after a + shard has unfollowed. If testRetentionLeaseIsAddedIfItDisappearsWhileFollowing + intercepts a renew request from other tests then we will never unlatch + and the test will time out. + + Closes #45192 + +commit 0887cbc964ffa8b4f4d746fbb39b686f91d6feb0 +Author: Nhat Nguyen +Date: Fri Nov 1 09:41:45 2019 -0400 + + Fix testForceMergeWithSoftDeletesRetentionAndRecoverySource (#48766) + + This test failure manifests the limitation of the recovery source merge + policy explained in #41628. If we already merge down to a single segment + then subsequent force merges will be noop although they can prune + recovery source. We need to adjust this test until we have a fix for the + merge policy. + + Relates #41628 + Closes #48735 + +commit 3c2054182358eaba9bceb7d3316b42af3b337da5 +Author: Armin Braun +Date: Sat Nov 2 20:42:29 2019 +0100 + + Cleanup Concurrent RepositoryData Loading (#48329) (#48834) + + The loading of `RepositoryData` is not an atomic operation. + It uses a list + get combination of calls. + This lead to accidentally returning an empty repository data + for generations >=0 which can never not exist unless the repository + is corrupted. + In the test #48122 (and other SLM tests) there was a low chance of + running into this concurrent modification scenario and the repository + actually moving two index generations between listing out the + index-N and loading the latest version of it. Since we only keep + two index-N around at a time this lead to unexpectedly absent + snapshots in status APIs. + Fixing the behavior to be more resilient is non-trivial but in the works. + For now I think we should simply throw in this scenario. This will also + help prevent corruption in the unlikely event but possible of running into this + issue in a snapshot create or delete operation on master failover on a + repository like S3 which doesn't have the "no overwrites" protection on + writing a new index-N. + + Fixes #48122 + +commit a22f6fbe3c5c49038c51e258a78fe67d8235fe0c +Author: Armin Braun +Date: Sat Nov 2 17:28:12 2019 +0100 + + Cleanup Redundant Futures in Recovery Code (#48805) (#48832) + + Follow up to #48110 cleaning up the redundant future + uses that were left over from that change. + +commit 4c707708778966ecc41e359c8ded4e6efcff96cf +Author: Nhat Nguyen +Date: Fri Nov 1 13:58:58 2019 -0400 + + Add debug log for CcrRetentionLeaseIT (#48820) + + testRetentionLeaseIsAddedIfItDisappearsWhileFollowing is still failing + although we already have several fixes. I think other tests interfere + and cause this test to fail. We can use the test scope to isolate them. + However, I prefer to add debug logs so we can find the source. + + Relates #45192 + +commit c24595e2ec722624dd91091ddd84169d1f1a9fb5 +Author: Jason Tedor +Date: Fri Nov 1 17:27:36 2019 -0400 + + Fix names of UBI-based Docker image build contexts + + This commit fixes the name of the UBI-based Docker image build contexts + to include "7" (to set us up for the future where we are likely to have + a ubi8-based image). + +commit e26d01e71f2687e74caf13d73a8fdd3ceb3cf400 +Author: Armin Braun +Date: Fri Nov 1 21:02:47 2019 +0100 + + Make CcrRepository#restore non-Blocking (#48814) (#48823) + + With the changes in #48110 there is no more need + to block a generic thread when waiting for the multi file transfer + in `CcrRepository`. + +commit 6c290ecaf72e933fd4b4f221a7083972bcca132c +Author: Lee Hinman +Date: Fri Nov 1 13:57:47 2019 -0600 + + Fix ilm/20_move_to_step basic moving to step (#48821) + + Previously this step moved to the forcemerge step, however, if the + machine running the test was fast enough, it would execute the + forcemerge and move to the next step (`segment-count`) so the comparison + would fail. This commit changes the step to be a step that will never go + anywhere else, the terminal step. + + Resolves #48761 + +commit c82ecb664c51d90c4649f08f72775425ac3644f6 +Author: Jason Tedor +Date: Fri Nov 1 15:11:18 2019 -0400 + + Do not wrap ingest processor exception with IAE (#48816) + + The problem with wrapping here is that it converts any exception into an + IAE, which we treat as a client error (400 status) whereas the exception + being wrapped here could be a server error (e.g., NPE). This commit + stops wrapping all ingest processor exceptions as IAEs. + +commit 0e89a986c9cda4d8479591b974191d1d1e342888 +Author: Jason Tedor +Date: Fri Nov 1 15:00:37 2019 -0400 + + Register UBI-based Docker image projects in build + + This commit registers the UBI-based Docker image projects in the build + so that their assemble tasks are executed when the top-level assemble + task is executed. + +commit 5ecde37a68401d219c5b7790804803f8e92d6036 +Author: Hendrik Muhs +Date: Fri Nov 1 19:39:35 2019 +0100 + + [7.x][Transform] decouple task and indexer (#48812) + + decouple TransformTask and ClientTransformIndexer. Interaction between the 2 classes are + now moved into a context class which holds shared information. + + relates #45369 + +commit 6ab4645f4e27dda7f4a86ecc8cdd874a52942f9e +Author: Mark Vieira +Date: Fri Nov 1 11:33:11 2019 -0700 + + [7.x] Introduce type-safe and consistent pattern for handling build globals (#48818) + + This commit introduces a consistent, and type-safe manner for handling + global build parameters through out our build logic. Primarily this + replaces the existing usages of extra properties with static accessors. + It also introduces and explicit API for initialization and mutation of + any such parameters, as well as better error handling for uninitialized + or eager access of parameter values. + + Closes #42042 + +commit 700a316bb32bb943b2e9bd240401c66c80e165f5 +Author: James Rodewig +Date: Fri Nov 1 12:37:24 2019 -0400 + + [DOCS] Reformat decimal digit token filter docs (#48722) + +commit 680999f246e0b1a31e0c63bac7e789800c2fd2f6 +Author: James Rodewig +Date: Fri Nov 1 11:54:13 2019 -0400 + + [DOCS] List `indices.lifecycle.poll_interval` as cluster-level (#48813) + + Lists `indices.lifecycle.poll_interval` with other cluster-level ILM + settings. + + Previously, it was included under index-level settings. + +commit f2d4c94a9cca7702f91212188ca4b633c82b09e3 +Author: Dimitris Athanasiou +Date: Fri Nov 1 16:53:05 2019 +0200 + + [7.x][ML] Deduplicate multi-fields for data frame analytics (#48799) (#48806) + + In the case multi-fields exist in the source index, we pick + all variants of them in our extracted fields detection for + data frame analytics. This means we may have multiple instances + of the same feature. The worse consequence of this is when the + dependent variable (for regression or classification) is also + duplicated which means we train a model on the dependent variable + itself. + + Now that #48770 is merged, this commit is adding logic to + only select one variant of multi-fields. + + Closes #48756 + + Backport of #48799 + +commit fd4ae697b898c21720c872552af6e925456c51df +Author: Tim Vernum +Date: Wed Sep 25 18:26:47 2019 +1000 + + Fix indentation of "except" in role mapping doc + + "except" is a type of rule, and should be indented accordingly. + +commit 3604add5c9e64634a0fc4effa4e0af497a640770 +Author: Dan Hermann +Date: Fri Nov 1 09:02:37 2019 -0500 + + [7.x] Validate monitoring username at parse time (#48774) + +commit 9a0a7ab95ace6c528ef7935548e80a65d69bd92a +Author: pulysak +Date: Fri Nov 1 16:15:14 2019 +0300 + + [DOCS] Fix typo in Index API reference docs (#48760) + +commit 80cde68af7453e491a4798e956bf830b715df79f +Author: Alexander Reelsen +Date: Fri Nov 1 14:02:42 2019 +0100 + + [DOCS] Remove unneeded // CONSOLE comments from snippets (#48763) + + Updates the docs README file to remove outdated `// CONSOLE` instructions + +commit 98a92275881e23585534a09949d98b107f6ac4d9 +Author: Andrei Dan +Date: Fri Nov 1 12:34:33 2019 +0000 + + Fix TimeSeriesLifecycleActionsIT.testRolloverAlreadyExists (#48747) (#48795) + + * ILM Test asserts on the same ilm/_explain output + + With the introduction of retryable steps subsequent ilm/_explain calls + can see the state of an ilm cycle move out of the error step. This test + made several assertions assuming that the cycle remains in the error + step so this commit changes the test to make one _explain call and have + all the asserts work on the same ilm state (so subsequent assumptions to + the cycle being in the error step are valid). + + * Drop unused field in test. + + (cherry picked from commit 44c74bb487151c886a08b27f32b13f7a72056997) + Signed-off-by: Andrei Dan + +commit 1f662e0b126b43caf17052e18a2ed36d0bfd8835 +Author: Dimitris Athanasiou +Date: Fri Nov 1 14:18:03 2019 +0200 + + [7.x][ML] Prevent fetching multi-field from source (#48770) (#48797) + + Aggregatable mutli-fields are at the moment wrongly mapped + as normal doc_value fields and thus they support fetching from + source. However, they do not exist in the source. This results + to failure to extract such fields. + + This commit fixes this bug. While a fix could be worked out + on top of the existing code, it is evident the extraction logic + has become difficult to understand and maintain. As we also + want to deduplicate multi-fields for data frame analytics, + it seemed appropriate to refactor the code to simplify and + better handle the extraction of multi-fields. + + Relates #48756 + + Backport of #48770 + +commit e1e9b23db8a1c24b12cbca83acdb90e0ad7638f1 +Author: Andrei Stefan +Date: Thu Oct 31 23:24:40 2019 -0400 + + Cleanup static instance in @AfterClass + +commit b9f4b3289258fdd6745ad3e377c512359bbf0e6f +Author: debadair +Date: Thu Oct 31 18:57:02 2019 -0700 + + [DOCS] Fix cross-doc link. (#48783) + + * [DOCS] Fix cross-doc link. + + * Fixed xref + +commit 40834c229f8434921207d3f1588e39feac4ee0a8 +Author: Lisa Cawley +Date: Thu Oct 31 18:22:08 2019 -0700 + + [7.x][DOCS] Copies ESMS monitoring details to Elasticsearch Reference (#48780) + +commit 457379e74e930f1c5fdce2867a6d51f3e108b133 +Author: debadair +Date: Thu Oct 31 18:04:07 2019 -0700 + + [DOCS] Edited Docker install & tweaked Docker compose file. (#47715) + + * [DOCS] Edited Docker install & tweaked Docker compose file. + + * Synced with Docker GS in SO + + * Incorporated review comments + +commit 2c73c7dfe3164aee0cbfb4863c379f36e874503f +Author: Andrei Stefan +Date: Thu Oct 31 17:44:34 2019 -0400 + + SQL: binary communication implementation for drivers and the CLI (#48261) + + * Introduce binary_format request parameter (binary.format for JDBC) to disable binary + communication between clients (jdbc/odbc) and server. + * for CLI - "binary" command line parameter (or -b) is introduced. Default value is "true". + * binary communication (cbor) is enabled by default + * disabling request parameter introduced for debugging purposes only + + (cherry picked from commit f96a5ca61cb9fad9ed59357320af20e669348ce7) + +commit 4be54402de5d0d272966b921ef51fcbbcd968489 +Author: Tal Levy +Date: Thu Oct 31 14:36:54 2019 -0700 + + [7.x] Add ingest info to Cluster Stats (#48485) (#48661) + + * Add ingest info to Cluster Stats (#48485) + + This commit enhances the ClusterStatsNodes response to include global + processor usage stats on a per-processor basis. + + example output: + + ``` + ... + "processor_stats": { + "gsub": { + "count": 0, + "failed": 0 + "current": 0 + "time_in_millis": 0 + }, + "script": { + "count": 0, + "failed": 0 + "current": 0, + "time_in_millis": 0 + } + } + ... + ``` + + The purpose for this enhancement is to make it easier to collect stats on how specific processors are being used across the cluster beyond the current per-node usage statistics that currently exist in node stats. + + Closes #46146. + + * fix BWC of ingest stats + + The introduction of processor types into IngestStats had a bug. + It was set to `null` and set as the key to the map. This would + throw a NPE. This commit resolves this by setting all the processor + types from previous versions that are not serializing it out to + `_NOT_AVAILABLE`. + +commit d0ead688c38a7173ffa90369fa0bf01a2c546e7f +Author: Lee Hinman +Date: Thu Oct 31 15:25:12 2019 -0600 + + [7.x] Fix TimeSeriesLifecycleActionsIT.testExplainFilters (#48… (#48776) + + This test used an index without an alias to simulate a failure in the + `check-rollover-ready` step. However, with #48256 that step + automatically retries, meaning that the index may not always be in + the ERROR step. + + This commit changes the test to use a shrink action with an invalid + number of shards so that it stays in the ERROR step. + + Resolves #48767 + +commit 99aedc844da94d7dd8e93b315d904082d517048b +Author: Ioannis Kakavas +Date: Thu Oct 31 23:05:12 2019 +0200 + + Copy http headers to ThreadContext strictly (#45945) (#48675) + + Previous behavior while copying HTTP headers to the ThreadContext, + would allow multiple HTTP headers with the same name, handling only + the first occurrence and disregarding the rest of the values. This + can be confusing when dealing with multiple Headers as it is not + obvious which value is read and which ones are silently dropped. + + According to RFC-7230, a client must not send multiple header fields + with the same field name in a HTTP message, unless the entire field + value for this header is defined as a comma separated list or this + specific header is a well-known exception. + + This commits changes the behavior in order to be more compliant to + the aforementioned RFC by requiring the classes that implement + ActionPlugin to declare if a header can be multi-valued or not when + registering this header to be copied over to the ThreadContext in + ActionPlugin#getRestHeaders. + If the header is allowed to be multivalued, then all such headers + are read from the HTTP request and their values get concatenated in + a comma-separated string. + If the header is not allowed to be multivalued, and the HTTP + request contains multiple such Headers with different values, the + request is rejected with a 400 status. + +commit 046f5bfd9f82d52fef525b0b7915dc550e31c26d +Author: Jason Tedor +Date: Thu Oct 31 15:28:30 2019 -0400 + + Fix complication in distribution/archives/build.gradle + + This commit fixes a missing brace in distribution/archives/build.gradle. + +commit 90d44375136e0b66608be59949ab2bad4710111a +Author: Jason Tedor +Date: Thu Oct 31 15:18:11 2019 -0400 + + Simplify JDK copy specification in archives build + + This commit simplifies the JDK copy specification in the archives build + so that it's a single line as opposed to an if/else with a repeated + body. This approach reduces the maintenance cost of this code. + +commit b1c50254986cf435d5bec4cba4e16717b56875eb +Author: Jason Tedor +Date: Thu Oct 31 15:11:44 2019 -0400 + + Package the JDK into jdk.app on macOS (#48765) + + This commit packages the bundled JDK into jdk.app on macOS to assist + with notarization there. + +commit 6da97350c8cd1ed0664875efd3a83eb71cd7404e +Author: William Brafford +Date: Thu Oct 31 15:10:17 2019 -0400 + + Don't drop user's MaxDirectMemorySize flag on jdk8/windows (#48657) + + * Always pass user-specified MaxDirectMemorySize + + We had been testing whether a user had passed a value for + MaxDirectMemorySize by parsing the output of "java -XX:PrintFlagsFinal + -version". If MaxDirectMemorySize equals zero, we set it to half of max + heap. The problem is that on Windows with JDK 8, a JDK bug incorrectly + truncates values over 4g and returns multiples of 4g as zero. In order + to always respect the user-defined settings, we need to check our input + to see if an "-XX:MaxDirectMemorySize" value has been passed. + + * Always warn for Windows/jdk8 ergo issue + + Even if a user has set MaxDirectMemorySize, they aren't future-proof for + this JDK bug. With this change, we issue a general warning for the + windows/JDK8 issue, and a specific warning if MaxDirectMemorySize is + unset. + +commit 34c2375417a7d1dca07c727b82cc7f4229a65367 +Author: Zachary Tong +Date: Thu Oct 31 12:38:20 2019 -0400 + + Add v7.4.3 version constant + +commit 6e6b939fc3e902ae78f89ec18ccaff94c281668d +Author: Mark Vieira +Date: Thu Oct 31 10:19:36 2019 -0700 + + Remove unnecessary logic for fixing generated POMs (#48721) + + This commit eliminates some custom logic we have in place for post-hoc + cleanup of POM files generated by Gradle. There were to main issues this + logic was meant to address: + + First, for dependencies marked as `transitive = false`, Gradle by + default creates a "wildcard" exclusion in the generated POM file. It + turns out that Ivy didn't handle these types of exclusions well, even + though they are perfectly valid and dealt with by Gradle and Maven as + expected. We've since confirmed that this issues is indeed resolved in + the most recent Ivy release (2.5.0-rc1) so going forward the suggestion + to folks consuming Elasticsearch dependencies with Ivy will be to use + this version. + + Second, earlier versions of Gradle would incorrectly assign compile + dependencies to the "runtime" scope in the publish POM file. This could + cause issues if the dependencies were indeed needed at compile time + because their APIs were exposed. This has since been fixed and these + dependencies are correctly marked as "compile" scope in the POM. + + Since these two issues have been resolved in their respective projects + we can eliminate this logic and all the supporting code, such as having + to create lots of "internal" configurations for tracking transitive + dependencies. + +commit 6412d0f528d4325f60a8bd1ec4d8017f583a0451 +Author: Deb Adair +Date: Thu Oct 31 09:40:32 2019 -0700 + + [DOCS] Remove coming tag from 7.4.2 RN backport. + +commit 7892ea8affb156a7551676e088783cd3bcca5af9 +Author: Yannick Welsch +Date: Thu Oct 31 16:24:16 2019 +0100 + + Fix testRecoveryClosedIndex (#48506) + + Fixes test failure: https://gradle-enterprise.elastic.co/s/lplb52nlth4re/tests/ftbzinkix5rme-2cwvhj3qg5qou + +commit 088988bb372ccf4989b4e05b1c8ab167bf98add1 +Author: Andrey Ershov +Date: Thu Oct 31 18:21:36 2019 +0300 + + GCS snapshot cleanup tool backport to 7.x (#48750) + + This is the backport of #45076 with dependent changes. + +commit b7559f23cc5ac9faecd0fbce9cffdfe4998627e2 +Author: Lisa Cawley +Date: Thu Oct 31 07:37:44 2019 -0700 + + [DOCS] Fixes PR#48055 in release notes (#48726) + +commit 4ecf234617d93b6b31f13646e5a8446a3f7d928b +Author: Alexander Reelsen +Date: Thu Oct 31 14:26:35 2019 +0100 + + Upgrade to joda 2.10.4 (#47805) + +commit 185e067442008914877111a0c4ed461ae78f54b6 +Author: emasab +Date: Thu Oct 31 14:15:04 2019 +0100 + + SQL: Failing Group By queries due to different ExpressionIds (#43072) + + Fix an issue that arises from the use of ExpressionIds as keys in a lookup map + that helps the QueryTranslator to identify the grouping columns. The issue is + that the same expression in different parts of the query (SELECT clause and GROUP BY clause) + ends up with different ExpressionIds so the lookup fails. So, instead of ExpressionIds + use the hashCode() of NamedExpression. + + Fixes: #41159 + Fixes: #40001 + Fixes: #40240 + Fixes: #33361 + Fixes: #46316 + Fixes: #36074 + Fixes: #34543 + Fixes: #37044 + + Fixes: #42041 + (cherry picked from commit 3c38ea555984fcd2c6bf9e39d0f47a01b09e7c48) + +commit 7ea74918e1f3d4ec9cdc1f56ab631229ab9985d9 +Author: Stéphane Campinas +Date: Thu Oct 31 14:39:44 2019 +0100 + + [DOCS] Fix typo in IndexFieldData.java comments (#48743) + +commit c358ecb5fb86710aece6edc2339e75dcf834ffeb +Author: Martijn van Groningen +Date: Thu Oct 31 14:22:55 2019 +0100 + + Don't preserve indices between enrich qa tests. + + This was added because it was suspected to cause the monitoring + enrich verification to fail, but that is not the case. + + See #48258 + +commit 3f7aafa42106ff7481d8d0d1e3d20591204e4c69 +Author: Peter Johnson +Date: Thu Oct 31 14:11:57 2019 +0100 + + [DOCS] Fix typo in synonym token filter docs (#48691) + +commit 3d5b1725a907d60a3d312c29ae61fd8cb55aaf3e +Author: James Rodewig +Date: Thu Oct 31 09:07:27 2019 -0400 + + [DOCS] Remove unneeded filter from common grams analyze ex (#48748) + +commit aa02174d53788a260b6ad27b877ffe248eb639e1 +Author: Brandon Morelli +Date: Thu Oct 31 05:30:02 2019 -0700 + + [DOCS] Fix typo in ILM policy definition docs (#48723) + + Removes an extra "by". + +commit ffe5d5417fee7a0053dc3dcc96402033edcff99c +Author: Andrei Dan +Date: Thu Oct 31 11:28:55 2019 +0000 + + ILM Make the `check-rollover-ready` step retryable (#48256) (#48740) + + This adds the infrastructure to be able to retry the execution of retryable + steps and makes the `check-rollover-ready` retryable as an initial step to + make the rollover action more resilient to transient errors. + + (cherry picked from commit 454020ac8acb147eae97acb4ccd6fb470d1e5f48) + Signed-off-by: Andrei Dan + +commit 0366c4d4a96135707657fc05c0d80bba23a2f66a +Author: kkewwei +Date: Thu Oct 31 18:55:22 2019 +0800 + + Faster access to INITIALIZING/RELOCATING shards (#47817) + + Today a couple of allocation deciders iterate through all the shards on a node + to find the `INITIALIZING` or `RELOCATING` ones, and this can slow down cluster + state updates in clusters with very high-density nodes holding many thousands + of shards even if those shards belong to closed or frozen indices. This commit + pre-computes the sets of `INITIALIZING` and `RELOCATING` shards to speed up + this search. + + Closes #46941 + Relates #48579 + + Co-authored-by: "hongju.xhj" + +commit d96976e2b1b917acc0ddff38a69239379530db92 +Author: Rory Hunter +Date: Thu Oct 31 10:48:55 2019 +0000 + + Improve resiliency to formatting JSON in server (#48706) + + Backport of #48553. Make a number of changes so that JSON in the server + directory is more resilient to automatic formatting. This covers: + + * Reformatting multiline JSON to embed whitespace in the strings + * Add helper method `stripWhitespace()`, to strip whitespace from a JSON + document using XContent methods. This is sometimes necessary where + a test is comparing some machine-generated JSON with an expected + value. + +commit eefa84bc94c39da9bd119a662c4882e696212281 +Author: Arvind Ramachandran +Date: Thu Oct 31 06:07:39 2019 -0400 + + Ignore dangling indices created in newer versions (#48652) + + Today it is possible that we import a dangling index that was created in a + newer version than one or more of the nodes in the cluster. Such an index would + prevent the older node(s) from rejoining the cluster if they were to briefly + leave it for some reason. This commit prevents the import of such dangling + indices. + + Fixes #34264 + +commit 00d3151eea0a877689ece10a49013503f269d662 +Author: Albert Zaharovits +Date: Thu Oct 31 10:51:00 2019 +0200 + + Document allow_restricted_indices for indices privileges (#47514) + + Document the allow_restricted_indices role descriptor field. + +commit 989467ca1e807efb56f7dbcfd3e9b4b3009b1cab +Author: Tanguy Leroux +Date: Thu Oct 31 10:43:43 2019 +0100 + + Add docker-compose based test fixture for Azure (#48736) + + This commit adds a new :test:fixtures:azure-fixture project which + provides a docker-compose based container that runs a AzureHttpFixture + Java class that emulates an Azure Storage service. + + The logic to emulate the service is extracted from existing tests and + placed in AzureHttpHandler into the new project so that it can be + easily reused. The :plugins:repository-azure project is an example + of such utilization. + + The AzureHttpFixture fixture is just a wrapper around AzureHttpHandler + and is now executed within the docker container. + + The :plugins:repository-azure:qa:microsoft-azure project uses the new + test fixture and the existing AzureStorageFixture has been removed. + +commit c3063c4e1f59c0f2193c96e24f4d71e7afe06ea0 +Author: David Roberts +Date: Thu Oct 31 09:18:04 2019 +0000 + + [ML] Make the URL of the ML C++ Ivy repo configurable (#48702) + + At present the ML C++ artifact is always downloaded from + S3. This change adds an option to configure the location. + + (The intention is to use a file:/// URL to pick up the + artifact built in a Docker container in ml-cpp PR builds + so that C++ changes that will break Java integration tests + can be detected before the ml-cpp PRs are merged.) + + Relates elastic/ml-cpp#766 + +commit fe8901b00bcd970c0ec1e2a44a34c352e3952c6c +Author: Yannick Welsch +Date: Thu Oct 31 09:44:56 2019 +0100 + + Return consistent source in updates (#48707) + +commit 5bea3898a90079de914314fcd47afaf36a699ae5 +Author: Ignacio Vera +Date: Thu Oct 31 08:46:57 2019 +0100 + + Add IndexOrDocValuesQuery to GeoPolygonQueryBuilder (#48449) (#48731) + +commit 51179f162ac3b0dbaec49267eb2676b337d6262d +Author: Jason Tedor +Date: Wed Oct 30 23:48:06 2019 -0400 + + Move ubi in UBI-based Docker image tags to the name (#48728) + + After some consideration, we are electing to make "ubi" part of the + image name instead of part of the tag. This commit implements that + change for the Elasticsearch UBI-based Docker images. + +commit 919596b2e80aaaf939f67641aecef823d0f58bdc +Author: Dimitris Athanasiou +Date: Thu Oct 31 02:41:00 2019 +0200 + + [7.x][ML] Move field extraction logic to its own package (#48709) (#48712) + + Moves common field extraction logic to its own package so that it can + be used both for anomaly detection and data frame analytics. + + In preparation for refactoring extraction fields to be simpler and to + support multi-fields properly. + + Backport of #48709 + +commit c7342dde29ae5819da607d8cc9e1334b7fe68095 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Oct 31 10:16:33 2019 +1100 + + Fix to release system resource after reading JKWSet file (#48666) (#48677) + + When we load a JSON Web Key (JWKSet) from the specified + file using JWKSet.load it internally uses IOUtils.readFileToString + but the opened FileInputStream is never closed after usage. + https://bitbucket.org/connect2id/nimbus-jose-jwt/issues/342 + + This commit reads the file and parses the JWKSet from the string. + + This also fixes an issue wherein if the underlying file changed, + for every change event it would add another file watcher. The + change is to only add the file watcher at the start. + + Closes #44942 + +commit a8767608483b749aa6f90cd9e71cd364e82400b0 +Author: debadair +Date: Wed Oct 30 15:55:43 2019 -0700 + + [DOCS] Add placeholder for 7.4.2 release notes (#48724) + +commit 56be621186ee1e1c376020d47cc15c1de3373c6e +Author: Jason Tedor +Date: Wed Oct 30 19:04:40 2019 -0400 + + Add UBI-based Docker images (#48710) + + This commit adds Docker images based on the UBI base image. + +commit 2d5291cf3ba056f4492ac97b8e88892e59f4e7ce +Author: Lee Hinman +Date: Wed Oct 30 16:11:27 2019 -0600 + + Un-AwaitsFix and enhance logging for testPolicyCRUD (#48719) + + * Un-AwaitsFix and enhance logging for testPolicyCRUD + + This removes the `AwaitsFix` and increases the test logging for + `SnapshotLifecycleServiceTests.testPolicyCRUD` in an effort to track + down the cause of #44997. + + * Remove unused import + +commit ae1ef5fd9227b62cf1cb59ba462d6c175360fe4a +Author: Julie Tibshirani +Date: Wed Oct 30 15:20:49 2019 -0700 + + Refactor unit tests for vector functions. (#48662) + + This PR performs the following changes: + * Split `ScoreScriptUtilsTests` into `DenseVectorFunctionTests` and + `SparseVectorFunctionTests`. This will make it easier to delete all sparse + vector function tests once we remove support on 8.x. + * As much as possible, break up the large test methods into individual tests + for each vector function (`cosineSimilarity`, `l2norm`, etc.). + +commit ede1681c5aae5fb6e7ac157daadf7eb6e318e91f +Author: Michael Basnight +Date: Wed Oct 30 14:41:05 2019 -0500 + + Remove old HLRC base test case (#48714) + + This commit finishes cleaning up the AbstractHlrcXContentTestCase work + and removes this class. All classes that were using this are now using + the updated base class. + + Ref #39745 + +commit eee4cfaa8ba33a7fe9a153b78ea8a064720b2090 +Author: Michael Basnight +Date: Wed Oct 30 13:26:41 2019 -0500 + + Refactor HLRC transform stats test (#48708) + + This test uses a deprecated base class, and this commit moves it over + to the new class. + + Ref #39745 + +commit d63e5772c01bdb55284662e5c37ba7bfa55f470a +Author: Michael Basnight +Date: Wed Oct 30 13:25:52 2019 -0500 + + Cleanup HLRC graph tests to use new test style (#48644) + + The old graph tests were duplicated a lot and used a deprecated parent + class. This commit cleans that up and removes one of the duplicated + tests. + + Ref #39745 + +commit ed2bb73de2ab488be252ce230d470e391c809b34 +Author: Lee Hinman +Date: Wed Oct 30 13:13:19 2019 -0600 + + Fix SnapshotLifecycleService logger (#48711) + + The logger was erroneously using the `SnapshotLifecycleMetadata` class + for its initialization, making it hard to target packages for logging + levels since `SnapshotLifecycleMetadata` is in a different package. + +commit f8ef402027e01d03e616a3a64403631dbdc79f3a +Author: Nhat Nguyen +Date: Wed Oct 30 14:08:35 2019 -0400 + + Do not warm up searcher in engine constructor (#48605) + + With this change, we won't warm up searchers until we externally refresh + an engine. We explicitly refresh before allowing reading from a shard + (i.e., move to post_recovery state) and during resetting. These + guarantees that we have warmed up the engine before exposing the + external searcher. + + Another prerequisite for #47186. + +commit c9ead80c317e25d18aa935fba62440eaf753c639 +Author: Benjamin Trent +Date: Wed Oct 30 13:27:29 2019 -0400 + + [7.x] [ML][Inference] separating definition and config object storage (#48651) (#48695) + + * [ML][Inference] separating definition and config object storage (#48651) + + This separates out the `definition` object from being stored within the configuration object in the index. + + This allows us to gather the config object without decompressing a potentially large definition. + + Additionally, `input` is moved to the TrainedModelConfig object and out of the definition. This is so the trained input fields are accessible outside the potentially large model definition. + +commit 0476f014bcdcd79e0c3e790f1d542817ba915d93 +Author: Martijn van Groningen +Date: Mon Oct 28 10:30:24 2019 +0100 + + Unmuted and fixed test. + + Multiple invocations are expected. + + see #48519 + +commit 72a601c47f0082400dc2a83a34affbc3f7b15ed1 +Author: Lee Hinman +Date: Wed Oct 30 09:46:35 2019 -0600 + + [7.x] Don't schedule SLM jobs when services have been stopped… (#48692) + + This adds a guard for the SLM lifecycle and retention service that + prevents new jobs from being scheduled once the service has been + stopped. Previous if the node were shut down the service would be + stopped, but a cluster state or local master election would cause a job + to attempt to be scheduled. This could lead to an uncaught + `RejectedExecutionException`. + + Resolves #47749 + +commit 13043219ac6b3a56fe86b763957d96a075427a1e +Author: Jason Tedor +Date: Wed Oct 30 11:25:27 2019 -0400 + + Fix specification for cluster.remote.connect (#48690) + + The docs specify that cluster.remote.connect disables cross-cluster + search. This is correct, but not fully accurate as it disables any + functionality that relies on remote cluster connections: cross-cluster + search, remote data feeds, and cross-cluster replication. This commit + updates the docs to reflect this. + +commit 0b062bbc8286636ebab313cba9823691d27e6523 +Author: James Rodewig +Date: Wed Oct 30 11:10:35 2019 -0400 + + [DOCS] Correct required file ext for user agent ingest processor (#48688) + + For the user agent ingest processor, custom regex files must end + with the `.yml` file extension. + + This corrects the docs which said the `.yaml` extension was required. + +commit 7c2f5c51b50e6368e0daa445b2cedacba0b6fc8e +Author: Martijn van Groningen +Date: Mon Oct 28 09:26:16 2019 +0100 + + Muted test + + See #48519 + +commit 36039706b5c3ef476a8e2e87cd00a957feed3561 +Author: Armin Braun +Date: Wed Oct 30 15:43:41 2019 +0100 + + Fix SnapshotShardStatus Reporting for Failed Shard (#48556) (#48687) + + Fixes the shard snapshot status reporting for failed shards + in the corner case of failing the shard because of an exception + thrown in `SnapshotShardsService` and not the repository. + We were missing the update on the `snapshotStatus` instance in + this case which made the transport APIs using this field report + back an incorrect status. + Fixed by moving the failure handling to the `SnapshotShardsService` + for all cases (which also simplifies the code, the ex. wrapping in + the repository was pointless as we only used the ex. trace upstream + anyway). + Also, added an assertion to another test that explicitly checks this + failure situation (ex. in the `SnapshotShardsService`) already. + + Closes #48526 + +commit 52e5ceb321eac8621885e231a00d04f64a70ab23 +Author: Armin Braun +Date: Wed Oct 30 14:36:30 2019 +0100 + + Restore from Individual Shard Snapshot Files in Parallel (#48110) (#48686) + + Make restoring shard snapshots run in parallel on the `SNAPSHOT` thread-pool. + +commit dbc05cd808ef52fb2808a3862d1f203ab167e2bf +Author: Dan Hermann +Date: Wed Oct 30 08:25:03 2019 -0500 + + Add option to split processor for preserving trailing empty fields (#48685) + +commit 77acbc4fa9dad562d01a00290824756e8fd36660 +Author: James Rodewig +Date: Wed Oct 30 08:40:11 2019 -0400 + + [DOCS] Reformat common grams token filter (#48426) + +commit 356066ce6a08f9f85dac94143d39c058829cfb5d +Author: Yannick Welsch +Date: Wed Oct 30 11:13:29 2019 +0100 + + Revert "Mute get-ccr-stats doctest (#48375)" + + This reverts commit f861927e8b8fc987949ce996a131a2d272b9646e. + +commit 01e326d2e3a56829f46fba18668dfdee6bb25b19 +Author: Armin Braun +Date: Wed Oct 30 10:10:48 2019 +0100 + + Fix ref count handling in Engine.failEngine (#48639) (#48646) + + We can run into an already closed store here and hence + throw on trying to increment the ref count => moving to + the guarded ref count increment + + closes #48625 + +commit f2d68934e8ebfb8f52dae0ec6ba413c1ae87bb42 +Author: Alpar Torok +Date: Wed Oct 30 10:39:34 2019 +0200 + + Only log vagrant output on failure (#48402) + +commit 9ed7352a123e28b4ecd8e2bde84b029c4277d23a +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Oct 30 14:19:54 2019 +1100 + + Add Sysprop to Adjust IO Buffer Size (#48267) (#48667) + + The 1MB IO-buffer size per transport thread is causing trouble in + some tests, albeit at a low rate. Reducing the number of transport + threads was not enough to fully fix this situation. + Allowing to configure the size of the buffer and reducing it by + more than an order of magnitude should fix these tests. + + Closes #46803 + +commit 1b64c1992a79513f5f980246986f28e24540ec90 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Oct 30 13:04:48 2019 +1100 + + Add owner flag parameter to the rest spec (#48500) + + This commit adds missing info about newly added + `owner` flag to the rest spec, also adds a rest test + for the same. + + Closes#48499 + +commit 89c65752dc633cd80c30133563ac1fef464c74d8 +Author: Julie Tibshirani +Date: Tue Oct 29 15:46:05 2019 -0700 + + Update the signature of vector script functions. (#48653) + + Previously the functions accepted a doc values reference, whereas they now + accept the name of the vector field. Here's an example of how a vector function + was called before and after the change. + + ``` + Before: cosineSimilarity(params.query_vector, doc['field']) + After: cosineSimilarity(params.query_vector, 'field') + ``` + + This seems more intuitive, since we don't allow direct access to vector doc + values and the the meaning of `doc['field']` is unclear. + + The PR makes the following changes (broken into distinct commits): + * Add new function signatures of the form `function(params.query_vector, + 'field')` and deprecates the old ones. Because Painless doesn't allow two + methods with the same name and number of arguments, we allow a generic `Object` + to be passed in to the function and decide on the behavior through an + `instanceof` check. + * Refactor the class bindings so that the document field is passed to the + constructor instead of the instance method. This allows us to avoid retrieving + the vector doc values on every function invocation, which gives a tiny speed-up + in benchmarks. + + Note that this PR adds new signatures for the sparse vector functions too, even + though sparse vectors are deprecated. It seemed simplest to understand (for both + us and users) to keep everything symmetric between dense and sparse vectors. + +commit 25724c5c46f65f9eb9433d1279b7d92276c4c904 +Author: Gordon Brown +Date: Tue Oct 29 15:44:04 2019 -0700 + + Adjust date parsing in ILM integration tests (#48648) + + The format returned by the API is not always parsable with + `Instant.parse()`, so this commit adjusts to parsing those dates as + `ISO_ZONED_DATE_TIME` instead, which appears to always parse the + returned value correctly. + +commit ef9275567aa9a406daa7d2e79c1135cf7fec2e41 +Author: Mark Vieira +Date: Tue Oct 29 14:46:12 2019 -0700 + + Reduce packaging test log output and introduce ready request interval (#48324) + +commit 7002ce1e9cce9695c0098ada427582dbf37b4abd +Author: James Rodewig +Date: Tue Oct 29 16:40:39 2019 -0400 + + [DOCS] Replace `_uid` refs in reindex slicing docs (#48649) + + PR #25543 removed the `_uid` field in favor of the `_id` field, + including for use in slicing. + + This removes an outdated reference to `_uid` in our reindex docs. + +commit 55d00cf2b1b01f28a18f408c7baa6ce53a67d1d9 +Author: Stuart Tettemer +Date: Tue Oct 29 14:41:15 2019 -0600 + + Scripting: fill in get contexts REST API (#48319) (#48602) + + Updates response for `GET /_script_context`, returning a `contexts` + object with a list of context description objects. The description + includes the context name and a list of methods available. The + methods list has the signature for the `execute` mathod and any + getters. eg. + ``` + { + "contexts": [ + { + "name" : "moving-function", + "methods" : [ + { + "name" : "execute", + "return_type" : "double", + "params" : [ + { + "type" : "java.util.Map", + "name" : "params" + }, + { + "type" : "double[]", + "name" : "values" + } + ] + } + ] + }, + { + "name" : "number_sort", + "methods" : [ + { + "name" : "execute", + "return_type" : "double", + "params" : [ ] + }, + { + "name" : "getDoc", + "return_type" : "java.util.Map", + "params" : [ ] + }, + { + "name" : "getParams", + "return_type" : "java.util.Map", + "params" : [ ] + }, + { + "name" : "get_score", + "return_type" : "double", + "params" : [ ] + } + ] + }, + ... + ] + } + ``` + + fixes: #47411 + +commit 50d7424e7d3219d90f9d5fcc3c8c92fd1241badd +Author: Gordon Brown +Date: Tue Oct 29 13:39:19 2019 -0700 + + Unmute and increase logging on flaky SLM tests (#48612) + + The failures in these tests have been remarkably difficult to track + down, in part because they will not reproduce locally. This commit + unmutes the flaky tests and increases logging, as well as introducing + some additional logging, to attempt to pin down the failures. + +commit 1de49d8a70f71b91bf35d1fd1d8e0b7d84aefe7d +Author: Christoph Büscher +Date: Tue Oct 29 20:55:48 2019 +0100 + + Remove Ranking Evaluation API experimental status (#48603) + + The API has been released long enough to remove the experimental status. + +commit 4bd514715d70e0bc4290065493ff9ce571053306 +Author: Gordon Brown +Date: Tue Oct 29 11:12:41 2019 -0700 + + Increase timeout in ILM doc test slightly (#48606) + + This assertBusy can occasionally time out on systems under heavy load, + such as CI, so this commit increases the timeout. + +commit 8b22e297ed84128c8faf6d113db1ab82287263b5 +Author: Andrei Dan +Date: Tue Oct 29 17:43:56 2019 +0000 + + ILM open/close steps are noop if idx is open/close (#48614) (#48640) + + The open and close follower steps didn't check if the index is open, + closed respectively, before executing the open/close request. + This changes the steps to check the index state and only perform the + open/close operation if the index is not already open/closed. + +commit be9df101bfded7894492b4e5e779f9c56b1187d4 +Author: Lisa Cawley +Date: Tue Oct 29 08:55:15 2019 -0700 + + [DOCS] Adds missing references to oidc realms (#48224) + +commit c6f4662038dae6a04bf74f88087d138af958227f +Author: Lisa Cawley +Date: Tue Oct 29 09:35:11 2019 -0700 + + [DOCS] Updates ML PRs in 7.4.1 release notes (#48600) + +commit cf235796c08caef72dc80a9a4efe3ba8acada5ca +Author: Gordon Brown +Date: Tue Oct 29 09:33:14 2019 -0700 + + Use more reliable "never run" cron pattern in tests (#48608) + + The cron schedule "1 2 3 4 5 ?" will run every May 4 at 03:02:01, which + may result in unnecessary test failures once a year. This commit + switches out uses of that schedule in tests for one which will never + execute (because it specifies a day which doesn't exist, Feb. 31). Also + factors the schedule out to a constant to make the intent clearer. + +commit d0cbbf9d58481abe6507098b2b9e2912a4fd71bc +Author: Daniel Andion +Date: Wed Oct 30 02:36:08 2019 +1100 + + SQL: [Docs] Typo in HAVING section (#48609) + + `HAVING` section code states `GROUP BY` instead of the appropriate keyword. + + (cherry picked from commit 9d505dc3db51e250fdf1b44e4d952dcd97bf1bc1) + +commit 7c944d26c5c29e8bf565e6254d9314061f1eac55 +Author: Przemysław Witek +Date: Tue Oct 29 16:20:56 2019 +0100 + + [7.x] Assert that the results of classification analysis can be evaluated using _evaluate API. (#48626) (#48634) + +commit 2a863ac8ff73b0d951357751034384e7f161bd45 +Author: Nhat Nguyen +Date: Tue Oct 29 10:13:46 2019 -0400 + + Fix testCleanUpCommitsWhenGlobalCheckpointAdvanced + + Relates #48559 + +commit b08cd058bce6be965a106ba0211f9e49663df192 +Author: Nhat Nguyen +Date: Tue Oct 29 08:56:06 2019 -0400 + + Greedily advance safe commit on new global checkpoint (#48559) + + Today we won't advance the safe commit on a new global checkpoint unless + the last commit can become safe. This is not great if we have more than + two commits as we can have a new safe commit earlier. + + Closes #4853 + +commit 09d68e7548207d0b3edae19258960cf2eecd39a1 +Author: Christoph Büscher +Date: Tue Oct 29 14:54:33 2019 +0100 + + Support `search_type` in Rank Evaluation API (#48542) (#48631) + + Adding support for the `search_type` request parameter to the Ranking Evaluation + API since this parameter can impact the ranking and the metric score and should + be choosen in the same way when evaluating the search as later in the real + search. + + Closes #48503 + +commit d004a560df5be968afb0a4f2ca732cdc1aaed2f7 +Author: Alpar Torok +Date: Tue Oct 29 14:47:21 2019 +0200 + + Upgrade to Gradle 5.6.3 (#48235) + + This PR upgrades to [5.6.3](https://github.com/gradle/gradle/releases/tag/v5.6.3). + Nothing particualrily of interest for us, just keeping up to date. + +commit aa70ff5ea4f2b2756c2a8085e39c2a122004c425 +Author: Jim Ferenczi +Date: Tue Oct 29 13:46:14 2019 +0100 + + Fix failures in ShuffleForcedMergePolicyTests#testDiagnostics (#48627) + + This commit fixes intermittent failures in ShuffleForcedMergePolicyTests#testDiagnostics + by setting a more restricted merge policy that ensures that extra merging will not happen + before the forced merge. + +commit a0362153e2d6e41d0d944a817bcef099f869e751 +Author: Ioannis Kakavas +Date: Tue Oct 29 14:18:59 2019 +0200 + + Update oauth2-oidc-sdk and nimbus-jose-jwt (#48537) (#48628) + + Update two dependencies for our OpenID Connect realm implementation + to their latest versions + +commit 08bf89b92bf8f0be12e9d3b62a6561d3e641b0dd +Author: Alpar Torok +Date: Tue Oct 29 14:05:34 2019 +0200 + + Work around incorrect lintian version on debian 10 (#48620) + + * Work around incorrect lintian version on debian 10 + + Relates to #48573 and intended as a work around to stop the CI failures + until we fix the image. + + * Need to pass --allow-downgrades + +commit c6abe58f635194e389396d91a43e73b979e66859 +Author: Jim Ferenczi +Date: Tue Oct 29 10:42:37 2019 +0100 + + Fix expectations in SearchAfter integration tests (#48372) + + This commit fixes the expectations of SearchAfterIT#shouldFail regarding the inner exceptions that should be thrown + when testing failures. The exception is sometimes wrapped in a QueryShardException so this change only checks that + the toString representation contains the expected message. + + Closes #43143 + +commit 6af3ce58f873d4ad9d573042f782c48b5bfcc7ec +Author: Yannick Welsch +Date: Tue Oct 29 12:09:59 2019 +0100 + + Filter on node id in AllocationIdIT (#48623) + + Makes the assertions more targeted. + + Relates #48529 + +commit da4654527bb21fc57cfc1610bba8538435042335 +Author: Rory Hunter +Date: Tue Oct 29 10:40:54 2019 +0000 + + Improve resiliency to auto-formatting in client (#48617) + + Backport of #48447. Make a number of changes so that code in the + `client` directory is more resilient to automatic formatting. This + covers: + + * Literal JSON handling: + * Reformatting multiline JSON to embed whitespace in the strings + * Remove string concatenation where JSON fits on a single line + * Use `String.format` for large documents with variable content + * Remove some erroneous doc refs in `QueryDSLDocumentationTests` + * Move some comments around to they aren't auto-formatted to a strange + place + +commit 61a4e24ecf1fb8e124c71f63c955a2ff6b15cded +Author: Rory Hunter +Date: Tue Oct 29 10:40:03 2019 +0000 + + Refine the auto-formatting settings (#48618) + + Backport of #48446. Update the Eclipse formatter settings as follows: + + * Split arguments for methods, constructors etc into separate lines when + they wrap + * Enable "joined wrapped lines", which basically means the formatter + will ignore existing line breaks and join and re-wrap statements. I + had originally hesitated about enabling this, but without it the + codebase's style won't be as consistent as it should be, defeating the + point of all this formatting business. + * Disable formatting between `// tag::` and `// end::` comments, which + are the code snippet markers for the docs. They need a different line + length, which isn't possible to configure separately. + * Detect and preserve aligned line comments at the end of consecutive + lines. + * And a bunch of other tweaks. + + Also update the contribution guidelines. + +commit 3c77c50f5f631276c35810746d6ed45719935cf5 +Author: Rory Hunter +Date: Tue Oct 29 10:39:34 2019 +0000 + + Improve resiliency to auto-formatting in libs, modules (#48619) + + Backport of #48448. Make a number of changes so that code in the libs and + modules directories are more resilient to automatic formatting. This covers: + + * Remove string concatenation where JSON fits on a single line + * Move some comments around to they aren't auto-formatted to a strange + place + +commit 028084ce232000d99ea33a60ed93132d67e8f939 +Author: Jim Ferenczi +Date: Tue Oct 29 09:00:04 2019 +0100 + + Add a new merge policy that interleaves old and new segments on force merge (#48533) + + This change adds a new merge policy that interleaves eldest and newest segments picked by MergePolicy#findForcedMerges + and MergePolicy#findForcedDeletesMerges. This allows time-based indices, that usually have the eldest documents + first, to be efficient at finding the most recent documents too. Although we wrap this merge policy for all indices + even though it is mostly useful for time-based but there should be no overhead for other type of indices so it's simpler + than adding a setting to enable it. This change is needed in order to ensure that the optimizations that we are working + on in # remain efficient even after running a force merge. + + Relates #37043 + +commit abddf5167289a9a23ec8b88b31e93776441fba27 +Author: lgypro <42293361+lgypro@users.noreply.github.com> +Date: Tue Oct 29 17:26:04 2019 +0800 + + [Docs] Fix syntax error leading to wrong doc ID (#48554) + + In order to index a document with id 2, the "&" should be replaced by "?" + +commit 82e25c4ac777453633a0b474c4b6c4a616d0a359 +Author: Ian Danforth <52720125+iandanforth-alation@users.noreply.github.com> +Date: Tue Oct 29 01:55:13 2019 -0700 + + [Docs] Fix typo in suggesters search API doc (#48477) + +commit 24f698523599d3e83a671db602ed00f83cbcc225 +Author: Tanguy Leroux +Date: Tue Oct 29 09:12:59 2019 +0100 + + Reduce allocations when draining HTTP requests bodies in repository tests (#48541) + + In repository integration tests, we drain the HTTP request body before + returning a response. Before this change this operation was done using + Streams.readFully() which uses a 8kb buffer to read the input stream, it + now uses a 1kb for the same operation. This should reduce the allocations + made during the tests and speed them up a bit on CI. + + Co-authored-by: Armin Braun + +commit 790cfc8ad2793034e770d0a1b56ffd60c64cc086 +Author: Yannick Welsch +Date: Tue Oct 29 08:09:06 2019 +0100 + + Fix upgraded_scroll test (#48525) + + I think the problem is that the master is trying to relocate the "upgraded_scroll" shard back to + the node on which it was previously allocated, but to which it can't be allocated now due to the + shard lock being held because of an in-progress scroll. As the master keeps on retrying and + retrying (and indefinitely tries so because max_retries does not apply to relocations, it blocks + any other lower-prioritized task from completing, which leads to the rolling upgrade tests failing + (see #48395). + + Closes #48395 + +commit 947f89a3a187e1052d0580b3cfb2364a1a7d7f42 +Author: Cris da Rocha +Date: Tue Oct 29 02:42:11 2019 +0100 + + Update troubleshooting.asciidoc (#48516) + +commit 4a076f5e92bfbc1553a81c723ad759d134530109 +Author: Ian Danforth <52720125+iandanforth-alation@users.noreply.github.com> +Date: Mon Oct 28 13:38:51 2019 -0700 + + [Doc] Fix typo in indices module docs (#48598) + +commit e5c6440a4f783dbe41cff39b407788056e3d179f +Author: Mark Vieira +Date: Mon Oct 28 12:11:55 2019 -0700 + + Simplify usage of Gradle Shadow plugin (#48478) (#48597) + + This commit simplifies and standardizes our usage of the Gradle Shadow + plugin to conform more to plugin conventions. The custom "bundle" plugin + has been removed as it's not necessary and performs the same function + as the Shadow plugin's default behavior with existing configurations. + + Additionally, this removes unnecessary creation of a "nodeps" artifact, + which is unnecessary because by default project dependencies will in + fact use the non-shadowed JAR unless explicitly depending on the + "shadow" configuration. + + Finally, we've cleaned up the logic used for unit testing, so we are + now correctly testing against the shadow JAR when the plugin is applied. + This better represents a real-world scenario for consumers and provides + better test coverage for incorrectly declared dependencies. + + (cherry picked from commit 3698131109c7e78bdd3a3340707e1c7b4740d310) + +commit 605500df7ea2f6b415ed2ad5aeefca0045ca45d6 +Author: Julie Tibshirani +Date: Mon Oct 28 11:57:20 2019 -0700 + + Add sparse vector deprecation to 7.6 migration docs. (#48435) + + This note was accidentally omitted from the deprecation PR. + +commit 6ea59dd428db4f20388e7be45c25d12a81eb7c92 +Author: Benjamin Trent +Date: Mon Oct 28 13:02:57 2019 -0400 + + [ML][Transforms] add wait_for_checkpoint flag to stop (#47935) (#48591) + + Adds `wait_for_checkpoint` for `_stop` API. + +commit 97f48168d93ff44cdabeada7c14740656db9476b +Author: Jason Tedor +Date: Mon Oct 28 12:30:43 2019 -0400 + + Bump bundled JDK to JDK 13.0.1+9 (#48587) + + This commit bumps the bundled JDK to 13.0.1+9. Since AdoptOpenJDK did + not release 13.0.1+9 for Windows, this commit also enables that the + bundled JDK version can vary by platform. + +commit 5021410165b77af60881df380a1e694738475bc6 +Author: Gordon Brown +Date: Mon Oct 28 09:24:38 2019 -0700 + + Retry on RepositoryException in SLM tests (#48548) + + Due to a bug, GETing a snapshot can cause a RespositoryException to be + thrown. This error is transient and should be retried, rather than + causing the test to fail. This commit converts those + RepositoryExceptions into AssertionErrors so that they will be retried + in code wrapped in assertBusy. + +commit 13ce179706900a8ceef84992e66775e8502a467e +Author: Lisa Cawley +Date: Mon Oct 28 08:42:09 2019 -0700 + + [DOCS] Re-enable code snippet testing in close anomaly detection job API (#48259) (#48585) + +commit 1ba57dbe08fa319070c71102a6e56f5a95f5ed7e +Author: Michael Basnight +Date: Mon Oct 28 09:54:39 2019 -0500 + + [Docs] add missing snapshot restore reference (#45256) + +commit c353ad71fe008aff8d672b183ab68d4f88515c8b +Author: Gordon Brown +Date: Mon Oct 28 07:37:52 2019 -0700 + + Wrap ResponseException in AssertionError in ILM/CCR tests (#48489) + + When checking for the existence of a document in the ILM/CCR integration + tests, `assertDocumentExists` makes an HTTP request and checks the + response code. However, if the repsonse code is not successful, the call + will throw a `ResponseException`. `assertDocumentExists` is often called + inside an `assertBusy`, and wrapping the `ResponseException` in an + `AssertionError` will allow the `assertBusy` to retry. + + In particular, this fixes an issue with `testCCRUnfollowDuringSnapshot` + where the index in question may still be closed when the document is + requested. + +commit 124f6d098b11942b470477b7acb82e50a2ae6e64 +Author: Marios Trivyzas +Date: Mon Oct 28 15:07:50 2019 +0100 + + SQL: [Tests] Renable CliSecurityIT (#48581) + + Seems that the issue has been fixed with: #48098 + + Closes: #48117 + (cherry picked from commit 470362361ffce794a6a12ce7a81a8029ec7d54de) + +commit e821a225803210928e37d20e4aa96b1b9feb8dfc +Author: David Turner +Date: Fri Oct 25 12:24:24 2019 +0100 + + Mute SecurityDocumentationIT#testGetUsers - see #48440 + +commit 7e30277a37f21967c19201707fe9abef52e3b939 +Author: Przemysław Witek +Date: Mon Oct 28 13:08:11 2019 +0100 + + Mute RegressionIT.testStopAndRestart (#48575) (#48576) + +commit 398c8028b383f24ce497d4b1ead44d158e94dcb2 +Author: Jason Tedor +Date: Mon Oct 28 07:57:44 2019 -0400 + + Remove locale provider picked up on backport + + This commit removes a locale provider that was inadvertently introduced + after a merge conflict during a backport. + +commit c4fbda3310c717252a45e895305f47a5092e62c4 +Author: Jason Tedor +Date: Mon Oct 28 07:45:56 2019 -0400 + + Introduce system JVM options (#48252) + + This commit moves JVM options that we are setting on behalf of the user + that we do not expect them to fiddle with out of the jvm.options + configuration file and into the JVM options parser. In this way, we + discourage fiddling with these settings, but more importantly, we ensure + that as we evolve or add to these settings that a user would pick these + pick instead of being left behind if they have a modified jvm.options + file and do not pick any new that come with the distribution. + +commit 53a22b8a8a453690878deb4cffe20478d9e16a3b +Author: Armin Braun +Date: Mon Oct 28 11:04:57 2019 +0100 + + Fix Validity of RepositoryDataTests Randomness (#48564) (#48566) + + Trivial point, but we were only testing shard generations + for a single shard here, accidentally, and not testing the + `null` generation case at all. + +commit 51c1b48b24c7ea336698c132221ea240df24ccec +Author: Alpar Torok +Date: Mon Oct 28 06:47:06 2019 +0200 + + Increase timeouts for packaging tests (#48528) + + This is in preparation to move to nested virtualization which is much slower + than the bare metal setup we use right now, but parallelizes better + resulting in a net win.t + +commit d235576452abd99d62cf600dcb86d706980895ca +Author: Alpar Torok +Date: Thu Oct 24 15:14:11 2019 +0300 + + Add a packagingTask for every os project (#48400) + + We no longer run the sample tests in CI, so it's safe to create a task + for every project. + This will make it easier to set them up in a matrix like fashion. + +commit 30389c66606008e775e8847fc10e60d1436c0555 +Author: Rory Hunter +Date: Sun Oct 27 16:06:23 2019 +0000 + + Improve SAML tests resiliency to auto-formatting (#48517) + + Backport of #48452. + + The SAML tests have large XML documents within which various parameters + are replaced. At present, if these test are auto-formatted, the XML + documents get strung out over many, many lines, and are basically + illegible. + + Fix this by using named placeholders for variables, and indent the + multiline XML documents. + + The tests in `SamlSpMetadataBuilderTests` deserve a special mention, + because they include a number of certificates in Base64. I extracted + these into variables, for additional legibility. + +commit 1ef87c9a689e34c6621c374bc26d30bb0ad0c757 +Author: Nhat Nguyen +Date: Fri Oct 25 17:31:35 2019 -0400 + + Refresh should not acquire readLock (#48414) + + Today, we hold the engine readLock while refreshing. Although this + choice simplifies the correctness reasoning, it can block IndexShard + from closing if warming an external reader takes time. The current + implementation of refresh does not need to hold readLock as + ReferenceManager can handle errors correctly if the engine is closed in + midway. + + This PR is a prerequisite that we need to solve #47186. + +commit 2e3db518c984b0e77124bc6da160052c7b148aaf +Author: Dan Hermann +Date: Fri Oct 25 16:22:11 2019 -0500 + + Do not reference values for filtered settings (#48066) (#48518) + +commit 5228956ecc5c7689d7191b51a01f30e584be902a +Author: Michael Basnight +Date: Fri Oct 25 15:22:44 2019 -0500 + + Add slices to delete and update by query in HLRC (#48420) + + The slices param was missing from both delete by query and update by + query in the HLRC request converters. This commit fixes the omission. + +commit 1dd7ab830c71acc901bb13f6ac1f9ee13d118dcc +Author: William Brafford +Date: Fri Oct 25 13:27:43 2019 -0400 + + Warn when MaxDirectMemorySize may be incorrect (Windows/JDK8 only issue) (#48365) + + Our JVM ergonomics extract max heap size from JDK PrintFlagsFinal output. + On JDK 8, there is a system-dependent bug where memory sizes are cast to + 32-bit integers. On affected systems (namely, Windows), when 1/4 of physical + memory is more than the maximum integer value, the output of PrintFlagsFinal + will be inaccurate. In the pathological case, where the max heap size would + be a multiple of 4g, the test will fail. + + The practical effect of this bug, beyond test failures, is that we may set + MaxDirectMemorySize to an incorrect value on Windows. This commit adds a + warning about this situation during startup. + +commit d27a307379274ed97a19ff37daf1ef391cc28e30 +Author: Shaunak Kashyap +Date: Fri Oct 25 09:35:13 2019 -0700 + + [DOCS] Remove extraneous comma in Enrich Stats API's JSON response (#48539) + +commit 45e42f4e1872bb8409f9270ba818abd122c6633f +Author: Tim Brooks +Date: Fri Oct 25 09:36:07 2019 -0600 + + Upgrade to Netty 4.1.43 (#48484) + + With this update we can remove the mitigation in our custom allocator + which forces heap buffer allocations. + +commit 7fc413c22c720afeb281ea5cb020f70ccccd1171 +Author: Jim Ferenczi +Date: Fri Oct 25 17:00:24 2019 +0200 + + Resolve the role query and the number of docs lazily (#48036) + + This commit ensures that the creation of a DocumentSubsetReader does not + eagerly resolve the role query and the number of docs that match. + We want to delay this expensive operation in order to ensure that we really + need this information when we build it. For this reason the role query and the + number of docs are now resolved on demand. This commit also depends on + https://issues.apache.org/jira/browse/LUCENE-9003 that will also compute the global + number of docs lazily. + +commit 2861088a5971aacd23d633c063578779f42994d2 +Author: Tanguy Leroux +Date: Fri Oct 25 17:41:34 2019 +0200 + + Mute RestClientMultipleHostsIntegTests.testCancelAsyncRequests (#48535) + + Relates https://github.com/elastic/elasticsearch/issues/45577 + +commit e9c8e4f6d1618b0f7f928ba5e4f33dade4e8b219 +Author: James Rodewig +Date: Fri Oct 25 10:30:52 2019 -0500 + + [DOCS] Fix note format in index suggestion docs (#48536) + +commit f5f1072824292fbc2d5ac21171c48b58806f9ee7 +Author: Tim Brooks +Date: Fri Oct 25 09:29:41 2019 -0600 + + Multiple remote connection strategy support (#48496) + + * Extract remote "sniffing" to connection strategy (#47253) + + Currently the connection strategy used by the remote cluster service is + implemented as a multi-step sniffing process in the + RemoteClusterConnection. We intend to introduce a new connection strategy + that will operate in a different manner. This commit extracts the + sniffing logic to a dedicated strategy class. Additionally, it implements + dedicated tests for this class. + + Additionally, in previous commits we moved away from a world where the + remote cluster connection was mutable. Instead, when setting updates are + made, the connection is torn down and rebuilt. We still had methods and + tests hanging around for the mutable behavior. This commit removes those. + + * Introduce simple remote connection strategy (#47480) + + This commit introduces a simple remote connection strategy which will + open remote connections to a configurable list of user supplied + addresses. These addresses can be remote Elasticsearch nodes or + intermediate proxies. We will perform normal clustername and version + validation, but otherwise rely on the remote cluster to route requests + to the appropriate remote node. + + * Make remote setting updates support diff strategies (#47891) + + Currently the entire remote cluster settings infrastructure is designed + around the sniff strategy. As we introduce an additional conneciton + strategy this infrastructure needs to be modified to support it. This + commit modifies the code so that the strategy implementations will tell + the service if the connection needs to be torn down and rebuilt. + + As part of this commit, we will wait 10 seconds for new clusters to + connect when they are added through the "update" settings + infrastructure. + + * Make remote setting updates support diff strategies (#47891) + + Currently the entire remote cluster settings infrastructure is designed + around the sniff strategy. As we introduce an additional conneciton + strategy this infrastructure needs to be modified to support it. This + commit modifies the code so that the strategy implementations will tell + the service if the connection needs to be torn down and rebuilt. + + As part of this commit, we will wait 10 seconds for new clusters to + connect when they are added through the "update" settings + infrastructure. + +commit d6d2edf324ee73ca4c635fe431013649179de904 +Author: Luca Cavanna +Date: Fri Oct 25 15:14:30 2019 +0200 + + Fix .tasks index strict mapping: parent_id should be parent_task_id (#48393) + + * Fix .tasks index strict mapping: parent_id should be parent_task_id + + The .tasks index has mappings that's strictly defined. `parent_task_id` + was defined as `parent_id` though which would cause an exception in case + a task is persisted that has a parent task id set. + + While at it, a couple of compiler warnings were addressed and a test + request builder was removed in favour of using its corresponding request. + + * increment version + +commit 9c48ed12bcdd54f03651888c51114a99e8c551aa +Author: Luca Cavanna +Date: Fri Oct 25 13:10:09 2019 +0200 + + Remove response search phase from ExpandSearchPhase (#48401) + + The expand phase is always created providing a function that builds + the next phase to be run, which has a single purpose: sending the + response back. Such small search phase is not necessary and causes some + issues when reporting search progress and counting the search phases + that need to be executed and that are already executed. We can simply + rather send back the response, without creating a specific phase for that. + +commit 055a0800eb91fb43c18473f27c0e7998df349440 +Author: Christoph Büscher +Date: Fri Oct 25 16:57:51 2019 +0200 + + [Docs] Mention reserved completion suggestion characters (#48445) + + We currently don't mention the three reserved characters anywhere. This change + adds a short note mentioning them + + Closes #48341 + +commit 06d2cc5cef6c414c720cb7c375788c9370754597 +Author: Tanguy Leroux +Date: Fri Oct 25 16:46:47 2019 +0200 + + Add missing azure error code (#48520) + + In #47176 we changed the internal HTTP server that emulates + the Azure Storage service so that it includes a response body + for injected errors. This fixed most of the issues reported in + #47120 but sadly I missed to map one error to its Azure + equivalent, and it triggered some CI failures today. + + Closes #47120 + +commit 9a89ed891e2aa95aec42480ae9e6bef72f849447 +Author: Andrei Stefan +Date: Fri Oct 25 17:26:22 2019 +0300 + + Detail the IDEs options for configuring the debug step (#48507) + + (cherry picked from commit 451d9cf06f1331ecfb6b4fa4acaed6fb43a1fcbb) + +commit 84a47a963217b6af983dba49d18bc7f5b1e7b8b2 +Author: Armin Braun +Date: Fri Oct 25 16:08:56 2019 +0200 + + Remove Outdated AwaitsFix (#48513) (#48522) + + This `AwaitsFix` was accidentally added after the test + was already fixed in #46594 => we can remove it. + +commit eb44a25899b6aec6a28ed9fece471a032a16d4ac +Author: Peter Dyson +Date: Fri Oct 25 13:11:31 2019 +0000 + + [DOCS] Reorder bullet items in CCS security docs (#48501) + + Adjust the last bullet item to be above the code block for better readability and to avoid it being skimmed over + +commit edab3748e9d12a512e7ab1f2bc2d9292bc4b9ed1 +Author: Armin Braun +Date: Fri Oct 25 15:03:23 2019 +0200 + + Remove Incorrect Assertion from SnapshotsInProgress (#47458) (#48514) + + This relates to the effort towards #46250. We added + tracking of the shard generation for successful + snapshots to `8.0`. + This assertion isn't correct though. While an `8.0` + master won't create an entry with sucess state and + a null shard generation it may still (on e.g. master + failover) send a success entry created by a 7.x master + with a `null` generation over the wire. + + Closes #47406 + +commit 3fb3397c12bd106a1185344a6cd9f2064fc79366 +Author: Christoph Büscher +Date: Fri Oct 25 13:34:44 2019 +0200 + + BlendedTermQuery's equals method should consider boosts (#48193) + + This changes the queries equals() method so that the boost factors for each term + are considered for the equality calculation. This means queries are only equal + if both their terms and associated boosts match. The ordering of the terms + doesn't matter as before, which is why we internally need to sort the terms and + boost for comparison on the first equals() call like before. Boosts that are + `null` are considered equal to boosts of 1.0f because topLevelQuery() will only + wrap into BoostQuery if boost is not null and different from 1f. + + Closes #48184 + +commit 486794f24dd6960f5a6f3388fa5f690c4ac2a8b7 +Author: Yannick Welsch +Date: Fri Oct 25 10:28:52 2019 +0200 + + Show task ID in source of persistent task state update (#48483) + + Relates #48395 + +commit b24bbd42964a5b07ca9c7ff0f8206e3525b5e9d4 +Author: Russ Cam +Date: Thu Oct 24 10:39:20 2019 +1000 + + Change policy_id to list type in slm.get_lifecycle (#47766) + + This commit changes the REST API spec slm.get_lifecycle's policy_id url part to be of type "list", in line with other REST API specs that accept a comma-separated list of values. + + Closes #47765 + +commit c0b545f3257ade7dd98683c9ab45957aa350e79c +Author: Tim Brooks +Date: Thu Oct 24 15:39:30 2019 -0600 + + Make BytesReference an interface (#48486) + + BytesReference is currently an abstract class which is extended by + various implementations. This makes it very difficult to use the + delegation pattern. The implication of this is that our releasable + BytesReference is a PagedBytesReference type and cannot be used as a + generic releasable bytes reference that delegates to any reference type. + This commit makes BytesReference an interface and introduces an + AbstractBytesReference for common functionality. + +commit d49958cef3a720dcaa395ec336917d1ee3430597 +Author: Michael Basnight +Date: Thu Oct 24 12:14:19 2019 -0500 + + Remove deprecated test from the HLRC tests (#48424) + + The AbstractHlrcWriteableXContentTestCase was replaced by a better test + case a while ago, and this is the last two instances using it. They have + been converted and the test is now deleted. + + Ref #39745 + +commit b2974e38161d7cc018267f88a6048525f2bd7b8d +Author: Julie Tibshirani +Date: Thu Oct 24 11:00:02 2019 -0700 + + Correct outdated information in _index docs. (#48436) + + This PR makes the following updates: + * Update the supported query types to include `prefix` and `wildcard`. + * Specify that queries accept index aliases. + * Clarify that when querying on a remote index name, the separator `:` must be + present. + +commit a4614daf4669930117240047a5362fe4cd19e3fc +Author: Jake Landis +Date: Thu Oct 24 12:07:02 2019 -0500 + + Allow more time for restart tests to reach yellow state. (#48434) (#48480) + + The testWatcher method will on occasion timeout waiting for + a yellow cluster state. This change increases the timeout + to 60s. + +commit c19379ef31df3dce98d9d91d33c657eee7416f95 +Author: Michael Basnight +Date: Thu Oct 24 09:04:57 2019 -0500 + + Remove random when using HLRC sync and async calls (#48211) + + This commit removes the randomization used by every execute call in the + high level rest tests. Previously every execute call, which can be many + calls per single test, would rely on a random boolean to determine if + they should use the sync or async methods provided to the execute + method. This commit runs the tests twice, using two different clusters, + both of them providing the value one time via a sysprop. This ensures + that the whole suite of tests is run using the sync and async code + paths. + + Closes #39667 + +commit b034153df7119caf3e26b4df96d58fd54b26e3fd +Author: Martijn van Groningen +Date: Thu Oct 24 15:33:30 2019 +0200 + + Change grok watch dog to be Matcher based instead of thread based. (#48346) + + There is a watchdog in order to avoid long running (and expensive) + grok expressions. Currently the watchdog is thread based, threads + that run grok expressions are registered and after completion unregister. + If these threads stay registered for too long then the watch dog interrupts + these threads. Joni (the library that powers grok expressions) has a + mechanism that checks whether the current thread is interrupted and + if so abort the pattern matching. + + Newer versions have an additional method to abort long running pattern + matching inside joni. Instead of checking the thread's interrupted flag, + joni now also checks a volatile field that can be set via a `Matcher` + instance. This is more efficient method for aborting long running matches. + (joni checks each 30k iterations whether interrupted flag is set vs. + just checking a volatile field) + + Recently we upgraded to a recent joni version (#47374), and this PR + is a followup of that PR. + + This change should also fix #43673, since it appears when unit tests + are ran the a test runner thread's interrupted flag may already have + been set, due to some thread reuse. + +commit 5ecfcdb162ffebd5b9de77e81463346088de585e +Author: Hendrik Muhs +Date: Thu Oct 24 15:16:20 2019 +0200 + + update warning about index names after transform rename (#48457) + + update warning about index names after transform rename + +commit fc1b4ad23cfa4532095c48e1759d29d3065fffa3 +Author: Dimitrios Liappis +Date: Thu Oct 24 15:52:56 2019 +0300 + + Mute testCCRUnfollowDuringSnapshot (#48464) + + tracked in #48461 + backport of #48462 + +commit 149537a165e93fdb8ca0745c3c93738c8b72a9a5 +Author: Przemysław Witek +Date: Thu Oct 24 14:18:43 2019 +0200 + + Assert that inference model has been persisted (#48332) (#48453) + +commit 4d0fb6e5517f957a7e6d928489a893c3ad8e9298 +Author: Dimitrios Liappis +Date: Thu Oct 24 14:53:12 2019 +0300 + + Mute testBasicTimeBasedRetenion (#48458) + + tracked in #48017 + backport of #48456 + +commit acf6d34d6917f0fb4ef57afcde7f06edd3cb017b +Author: Yannick Welsch +Date: Thu Oct 24 13:29:59 2019 +0200 + + Always use last properly persisted metadata as previous state (#47779) + + On data-only nodes we were not using the last persisted cluster state as base point to compute + what needed storage, but the last applied cluster state (but not necessarily properly persisted) + instead. + +commit ba1c13c47d752619759de0d227763a2a5b4728bd +Author: Hendrik Muhs +Date: Thu Oct 24 12:18:51 2019 +0200 + + [Transform] do not fail checkpoint creation due to global checkpoint mismatch (#48423) + + Take the max if global checkpoints mismatch instead of throwing an exception. It turned out global + checkpoints can mismatch by design + + fixes #48379 + +commit 65c58ed594bf10d06df9a993cbcf84aaeed94607 +Author: Yannick Welsch +Date: Thu Oct 24 12:08:21 2019 +0200 + + Use _type in op_type REST test (#48418) + + Closes #48396 + +commit 50518359fe52e8f5902f557a52987002ea8b4064 +Author: David Turner +Date: Thu Oct 24 09:43:46 2019 +0200 + + Fix relocating shards size calculation (#48421) + + In #48392 we added a second computation of the sizes of the relocating shards + in `canRemain()` but passed the wrong value for `subtractLeavingShards`. This + fixes that. It also removes some unnecessary logging in a test case added in + the same commit. + +commit dc5c31d67a714d91f05cf23d4ac30089f0ab34f2 +Author: Jim Ferenczi +Date: Thu Oct 24 09:25:50 2019 +0200 + + Add a deprecation warning regarding allocation awareness in search request (#48351) + + This is a follow up of https://github.com/elastic/elasticsearch/issues/43453 where we added + a system property to disallow allocation awareness in search requests. Since search requests + will no longer check the allocation awareness attributes for routing in the next major version, + this change adds a deprecation warning on any setup that uses these attributes. + + Relates #43453 + +commit c6b733f1b44ec1430b8c07e11743e5b82f9b2ee8 +Author: Ioannis Kakavas +Date: Thu Oct 24 09:51:08 2019 +0300 + + Add populate_user_metadata in OIDC realm (#48357) (#48438) + + Make populate_user_metadata configuration parameter + available in the OpenID Connect authentication realm + + Resolves: #48217 + +commit 05324b7f0368f053a5fe830b47199332c44322bc +Author: Martijn van Groningen +Date: Thu Oct 24 08:32:53 2019 +0200 + + Muted verifying monitoring integration in enrich integration test. + + Relates to #48258 + +commit 4375316b9dafe72c295ecea79d3da0357e1f1557 +Author: Julie Tibshirani +Date: Wed Oct 23 18:52:22 2019 -0700 + + Make sure to list the 7.5 migration docs. + +commit 2664cbd20bb2ef647d5eb93e93761391e6ea4068 +Author: Julie Tibshirani +Date: Wed Oct 23 16:35:03 2019 -0700 + + Deprecate the sparse_vector field type. (#48368) + + We have not seen much adoption of this experimental field type, and don't see a + clear use case as it's currently designed. This PR deprecates the field type in + 7.x. It will be removed from 8.0 in a follow-up PR. + +commit 9e9533f717a8ee920d45cbddb181b57615eed3e5 +Author: Mayya Sharipova +Date: Wed Oct 23 17:19:15 2019 -0400 + + Correct syntax from backport + + User older format of map + + Relates to #48425 + +commit 975dbecfa9ab964a9a0f0bfc76017cd0daba1333 +Author: Mayya Sharipova +Date: Wed Oct 23 16:45:10 2019 -0400 + + Correct rewritting of script_score query (#48425) + + Previously there was a bug when an query inside script_score query + was rewritten. If min_score was not set and was equal to null, + we were converting it to float value which resulted to NPE. + This commit corrects this. + + Closes #48081 + +commit 06dc1fbd96cff77adf3baf15669906bc5bc50b18 +Author: James Rodewig +Date: Wed Oct 23 15:06:18 2019 -0500 + + [DOCS] Reformat ASCII folding token filter docs (#48143) + +commit 8163e0a9e5aad801c1a9e2e3363d5a6936bc920a +Author: Igor Motov +Date: Thu Oct 10 21:37:03 2019 +0400 + + Mute XPackRestIT security/authz/14_cat_indices + + Mutes "Test empty request while single authorized closed index" + + Tracked by #47875 + +commit bdbc353deaa818ccba5bdb91fabe036c2a46c5a7 +Author: Igor Motov +Date: Wed Oct 23 10:31:32 2019 -0400 + + Geo: improve handling of out of bounds points in linestrings (#47939) + + Brings handling of out of bounds points in linestrings in line with + points. Now points with latitude above 90 and below -90 are handled + the same way as for points by adjusting the longitude by moving it by + 180 degrees. + + Relates to #43916 + +commit 41116eb7eaded6fee8332002a95c96f5d8be1a9c +Author: Jim Ferenczi +Date: Wed Oct 23 20:01:42 2019 +0200 + + Do not throw errors on unknown types in SearchAfterBuilder (#48147) + + * Do not throw errors on unknown types in SearchAfterBuilder + + The support for BigInteger and BigDecimal was added for XContent in + https://github.com/elastic/elasticsearch/pull/32888. However the SearchAfterBuilder + xcontent parser doesn't expect them to be present so it throws an AssertionError. + This change fixes this discrepancy by changing the AssertionError into an + IllegalArgumentException that will not cause the node to die when thrown. + + Closes #48074 + +commit 96556d72cc491f2ad2f7a04cda50aec40c64eeb2 +Author: Jim Ferenczi +Date: Wed Oct 23 19:57:37 2019 +0200 + + Add a known issue to the release notes of 7.4.0 (#48373) + + A [bug](https://github.com/elastic/elasticsearch/issues/48358) in 7.4.0 prevents + the activation of the search slow log. This change adds an entry in the release + notes to warn users to not activate it in this version. + + Relates #48358 + +commit 892264a97ae9012131df8c0411ddd04433b8e8da +Author: Tom Callahan +Date: Wed Oct 23 11:25:25 2019 -0400 + + Add versions 7.4.2 and 6.8.5 + +commit c783a20560741b5821d4f03162e1d7e59f6f4de3 +Author: David Turner +Date: Wed Oct 23 17:58:38 2019 +0200 + + Handle negative free disk space in deciders (#48392) + + Today it is possible that the total size of all relocating shards exceeds the + total amount of free disk space. For instance, this may be caused by another + user of the same disk increasing their disk usage, or may be due to how + Elasticsearch double-counts relocations that are nearly complete particularly + if there are many concurrent relocations in progress. + + The `DiskThresholdDecider` treats negative free space similarly to zero free + space, but it then fails when rendering the messages that explain its decision. + This commit fixes its handling of negative free space. + + Fixes #48380 + +commit 640d7416b14a5fe0700b00407c5a2f3d61b59712 +Author: James Rodewig +Date: Wed Oct 23 12:09:26 2019 -0500 + + [DOCS] Change prev version to 7.5 in upgrade docs (#48415) + +commit cf175da5a9a155dbfacdc56f5262c89e3d4e51d5 +Author: Jake Landis +Date: Wed Oct 23 11:21:39 2019 -0500 + + Ensure SLM stats does not block an in-place upgrade from 7.4 (… (#48411) + + 7.5+ for SLM requires [stats] object to exist in the cluster state. + When doing an in-place upgrade from 7.4 to 7.5+ [stats] does not exist + in cluster state, result in an exception on startup [1]. + + This commit moves the [stats] to be an optional object in the parser + and if not found will default to an empty stats object. + + [1] Caused by: java.lang.IllegalArgumentException: Required [stats] + +commit 19afe3f84c54d08a1e7428929698a3c9df5cb2fe +Author: James Rodewig +Date: Wed Oct 23 10:54:53 2019 -0500 + + [DOCS] Remove duplicate links for ingest processor overview (#48394) + +commit 81ef72d3efbe8a2be1c045179357c7a625b3c1b4 +Author: Adrien Grand +Date: Wed Oct 23 17:48:17 2019 +0200 + + Lucene#asSequentialBits gets the leadCost backwards. (#48335) (#48403) + + The comment says it needs random-access, but it passes `Long#MAX_VALUE` as the + lead cost, which forces sequential access, it should pass `0` instead. I took + advantage of this fix to improve the logic to leverage an estimation of the + number of times that `Bits#get` gets called to make better decisions. + +commit aaa6209be61c9670cca59d69b312b2c938f0d0e0 +Author: Przemyslaw Gomulka +Date: Wed Oct 23 17:39:38 2019 +0200 + + [7.x] [Java.time] Calculate week of a year with ISO rules BACKPORT(#48209) (#48349) + + Reverting the change introducing IsoLocal.ROOT and introducing IsoCalendarDataProvider that defaults start of the week to Monday and requires minimum 4 days in first week of a year. This extension is using java SPI mechanism and defaults for Locale.ROOT only. + It require jvm property java.locale.providers to be set with SPI,COMPAT + + closes #41670 + backport #48209 + +commit 9c75f14a9f2b22362c1dda7104ef19c8d1a9cb74 +Author: James Rodewig +Date: Wed Oct 23 09:38:22 2019 -0500 + + [DOCS] Reformat classic token filter docs (#48314) + +commit e1dd0e753db0c3e3972bff5a7005f97ffe74be6b +Author: Tanguy Leroux +Date: Wed Oct 23 16:54:23 2019 +0200 + + Differentiate service account tokens in GCS tests (#48382) + + This commit changes the test so that each node use a specific + service account and private key. It also changes how unique + request ids are generated for refresh token request using the + token itself, so that error count will be specific per node (each + node should execute a single refresh token request as tokens + are valid for 1 hour). + +commit 852622d970c17e0d01031eb44a814564246b6e9f +Author: James Rodewig +Date: Wed Oct 23 09:36:31 2019 -0500 + + [DOCS] Remove binary gendered language (#48362) + +commit cece5f24f704fc9a436b5ff9b4d386edf63c08a5 +Author: Ioannis Kakavas +Date: Wed Oct 23 17:24:04 2019 +0300 + + Add sections in SAML Troubleshooting (#47964) (#48387) + + - Section about the case where the `principal` user property can't + be mapped. + - Section about when the IdP SAML metadata do not contain a + SingleSignOnService that supports HTTP-Redirect binding. + + Co-Authored-By: Lisa Cawley + Co-Authored-By: Tim Vernum + +commit 834f2b45464cdcbe0de1135e3f06078f7b34edc2 +Author: Ioannis Kakavas +Date: Wed Oct 23 17:23:50 2019 +0300 + + Add brackets where necessary in error messages (#48140) (#48386) + + This commit attempts to help error readability by adding brackets + where applicable/missing in saml errors. + +commit 3be6135ceb5afdb83023c19c60aa545fa8666bba +Author: Alan Woodward +Date: Wed Oct 23 15:19:44 2019 +0100 + + Mute get-ccr-stats doctest (#48375) + + This test is failing frequently, due to #47718 + +commit f573cd6a2c4566f4854c23b1363d5848345af6e2 +Author: Tom Callahan +Date: Wed Oct 23 09:19:31 2019 -0400 + + [DOCS] Add 7.4.1 Release Notes (#48359) + +commit cb9f45ad78ddec655e19540181e402800fd3fcff +Author: Alpar Torok +Date: Wed Oct 23 13:27:14 2019 +0300 + + Always publish a build scan in CI (#48348) + + * Always publish a build scan in CI + + This PR changes the build scan configuration to alwasy publisha build + scan when running in our CI. + + We should alkready be passing these env vars into the Vagrant VM so this + will make it produce a build scan too. + + The old properties to accept build scan ToS on the public server are + thus no longer relevant and will be cleaned up from the Jenkins config + once this is merged. + + * Pass env vars to vagrant VM + * Enable running in parallel in the VM + * Add job name and build nomber as custom values + +commit 79014057f485fbb2a29b4d38be35b25809afe185 +Author: Alpar Torok +Date: Wed Oct 23 13:22:15 2019 +0300 + + Read build and runtime java from properties file (#48355) + + This PR changes the PS1 script that starts os tests for the packaging + test matrix to match the bash script we use on Linux in terms of reading + the runtime and build java versions. + + Relates to elastic/infra#11593 + +commit 7215201406fa11711b9980b0e83d7e943b3f5a4f +Author: Armin Braun +Date: Wed Oct 23 10:58:26 2019 +0100 + + Track Shard-Snapshot Index Generation at Repository Root (#48371) + + This change adds a new field `"shards"` to `RepositoryData` that contains a mapping of `IndexId` to a `String[]`. This string array can be accessed by shard id to get the generation of a shard's shard folder (i.e. the `N` in the name of the currently valid `/indices/${indexId}/${shardId}/index-${N}` for the shard in question). + + This allows for creating a new snapshot in the shard without doing any LIST operations on the shard's folder. In the case of AWS S3, this saves about 1/3 of the cost for updating an empty shard (see #45736) and removes one out of two remaining potential issues with eventually consistent blob stores (see #38941 ... now only the root `index-${N}` is determined by listing). + + Also and equally if not more important, a number of possible failure modes on eventually consistent blob stores like AWS S3 are eliminated by moving all delete operations to the `master` node and moving from incremental naming of shard level index-N to uuid suffixes for these blobs. + + This change moves the deleting of the previous shard level `index-${uuid}` blob to the master node instead of the data node allowing for a safe and consistent update of the shard's generation in the `RepositoryData` by first updating `RepositoryData` and then deleting the now unreferenced `index-${newUUID}` blob. + __No deletes are executed on the data nodes at all for any operation with this change.__ + + Note also: Previous issues with hanging data nodes interfering with master nodes are completely impossible, even on S3 (see next section for details). + + This change changes the naming of the shard level `index-${N}` blobs to a uuid suffix `index-${UUID}`. The reason for this is the fact that writing a new shard-level `index-` generation blob is not atomic anymore in its effect. Not only does the blob have to be written to have an effect, it must also be referenced by the root level `index-N` (`RepositoryData`) to become an effective part of the snapshot repository. + This leads to a problem if we were to use incrementing names like we did before. If a blob `index-${N+1}` is written but due to the node/network/cluster/... crashes the root level `RepositoryData` has not been updated then a future operation will determine the shard's generation to be `N` and try to write a new `index-${N+1}` to the already existing path. Updates like that are problematic on S3 for consistency reasons, but also create numerous issues when thinking about stuck data nodes. + Previously stuck data nodes that were tasked to write `index-${N+1}` but got stuck and tried to do so after some other node had already written `index-${N+1}` were prevented form doing so (except for on S3) by us not allowing overwrites for that blob and thus no corruption could occur. + Were we to continue using incrementing names, we could not do this. The stuck node scenario would either allow for overwriting the `N+1` generation or force us to continue using a `LIST` operation to figure out the next `N` (which would make this change pointless). + With uuid naming and moving all deletes to `master` this becomes a non-issue. Data nodes write updated shard generation `index-${uuid}` and `master` makes those `index-${uuid}` part of the `RepositoryData` that it deems correct and cleans up all those `index-` that are unused. + + Co-authored-by: Yannick Welsch + Co-authored-by: Tanguy Leroux + +commit 50f565b158573dd1142be46b8e93e1a9e55953f6 +Author: Jim Ferenczi +Date: Wed Oct 23 10:19:34 2019 +0200 + + SearchSlowLog uses a non thread-safe object to escape json (#48363) + + This commit fixes the usage of JsonStringEncoder#quoteAsUTF8 in the SearchSlowLog. + JsonStringEncoder#getInstance should always be called to get a thread local object + but this assumption was broken by #44642. This means that any slow log can throw + an AIOOBE since it uses the same byte array concurrently. + + Closes #48358 + +commit 4790ee4c329738ed0dde6e885b5180935b821322 +Author: Tanguy Leroux +Date: Tue Oct 22 11:40:24 2019 +0200 + + Reenable azure repository tests and remove some randomization in http servers (#48283) + + Relates #47948 + Relates #47380 + +commit c1f6aff5bb383e8522347541e077874f7e45cbfd +Author: Tim Brooks +Date: Tue Oct 22 19:50:31 2019 -0600 + + Remove default netty allocator empty assertions (#48356) + + This commit removes a problematic assertion that the netty default + allocator is not used. This assertion is problematic because any other + test can cause this task to fail by touching the default allocator. We + assert that we are using heap buffers in the channel. + +commit e45b0cd7e3afe85068fe763f1b0b3607bd172385 +Author: James Rodewig +Date: Tue Oct 22 12:27:31 2019 -0500 + + [DOCS] Sort cluster API docs alphabetically (#48198) + +commit 8a02a5fc7d16608ec632fa96179b38247cbd9894 +Author: Armin Braun +Date: Tue Oct 22 17:17:09 2019 +0100 + + Simplify Shard Snapshot Upload Code (#48155) (#48345) + + The code here was needlessly complicated when it + enqueued all file uploads up-front. Instead, we can + go with a cleaner worker + queue pattern here by taking + the max-parallelism from the threadpool info. + + Also, I slightly simplified the rethrow and + listener (step listener is pointless when you add the callback in the next line) + handling it since I noticed that we were needlessly rethrowing in the same + code and that wasn't worth a separate PR. + +commit b4ae207e1ef579cea4bc87099fe5123bbbf56715 +Author: Alpar Torok +Date: Tue Oct 22 17:03:07 2019 +0300 + + Relative paths, jornalctl in additional logs (#48276) + + * Relative paths, jornalctl in additional logs + + This PR fixes the archive generation to preserve the paths relative to + the project directory. + It also fixes calling journalctl to get the system log. + * explicitly remove the file we are building + * Skip files locked on windows + * Extended readability probe + * Try to read the file ahead on windows + * Make the tar a best effort + * Catch all exceptions + +commit 2f162a98a05bc42d26cd7a57d4660fde39122a25 +Author: Brandon DeVault +Date: Thu Oct 17 14:59:21 2019 -0400 + + [DOCS] Added the voting node.role value (#48162) + + * [DOCS] Added the voting node.role value + + With the elasticsearch.yml option "node.voting_only: true", if you perform a "_cat/nodes" there is a new option for `v` that will display if the node is a voting node. + + * [DOCS] Added the voting node.role value + + Updated "voting node" to read "voting-only node" + +commit e931fcd331a578c1104ce48ce9e2d58e11af90c1 +Author: James Rodewig +Date: Tue Oct 22 07:52:37 2019 -0500 + + [DOCS] Add placeholder for 7.4.1 release notes (#48316) + +commit c41951c6b3d15885e4f7e6889f62ca8af1ce782c +Author: Alexandre Fonseca +Date: Tue Oct 22 13:48:24 2019 +0200 + + [Docs] Fix opType options in IndexRequest API example. (#48290) + +commit 5ae7453878e0564605ae153d23af81e105c34be7 +Author: Hendrik Muhs +Date: Tue Oct 22 13:16:12 2019 +0200 + + [7.6][Transform] blacklist continuous transform tests if upgraded from 7.2.x (#48344) + + blacklist continuous transform tests if upgraded from 7.2.x + + fixes #48336 + +commit 60d8ecb2b74643188c94b620d2be3dcbf99aa1ae +Author: Przemysław Witek +Date: Tue Oct 22 12:45:50 2019 +0200 + + Mute ClassificationIT tests (#48338) (#48339) + +commit 24e43dfa3425468aa776679a7ffc8fb5180406ca +Author: Ioannis Kakavas +Date: Tue Oct 22 12:49:01 2019 +0300 + + [7.x] Refactor FIPS BootstrapChecks to simple checks (#47499) (#48333) + + FIPS 140 bootstrap checks should not be bootstrap checks as they + are always enforced. This commit moves the validation logic within + the security plugin. + The FIPS140SecureSettingsBootstrapCheck was not applicable as the + keystore was being loaded on init, before the Bootstrap checks + were checked, so an elasticsearch keystore of version < 3 would + cause the node to fail in a FIPS 140 JVM before the bootstrap check + kicked in, and as such hasn't been migrated. + + Resolves: #34772 + +commit aa29567e11495bbb1f622ee9db25eb8f479699b6 +Author: Przemysław Witek +Date: Tue Oct 22 11:34:09 2019 +0200 + + [7.x] Fix assignment + + Backport of https://github.com/elastic/elasticsearch/pull/48216 + +commit 144e8a0f1a634de901bbe281418085d8c7316203 +Author: Alpar Torok +Date: Tue Oct 22 11:16:16 2019 +0300 + + Use an env var for the classpath of jar hell task (#48240) + + The classpath for some project could outgrow the max allowed command + line on Windows. Using an env var is not fault proof, but give more + breathing room + +commit 3233b59b6810ae22e95349a4a14f40836b697bba +Author: Andrei Stefan +Date: Tue Oct 22 10:13:10 2019 +0300 + + Add "format" to "range" queries resulted from optimizing a logical AND (#48073) + + (cherry picked from commit 020939a9bd5b34c6d540faa8b3a67b740d661be3) + +commit 0656207e9a4f839f096bde73aabe657f7906bb13 +Author: Hendrik Muhs +Date: Tue Oct 22 09:00:08 2019 +0200 + + [DOCS][Transform] document limitation regarding rolling upgrade with 7.2, 7.3 (#48118) + + adds a limitation about rolling upgrade from 7.2 or 7.3. and fixes a problem with renamed preferences + +commit 1cb3b0cc0d5d57d46ea72c8b219a767cb30b6991 +Author: Hendrik Muhs +Date: Tue Oct 22 08:58:02 2019 +0200 + + [7.6][Transform] separate old and mixed rolling upgrade tests (#48302) + + separates rolling upgrade tests for transforms created on old and mixed clusters and disable + testing transforms on mixed clusters for <7.4. + +commit bbe50eca728f1029a3a322abde6fdaa91121343f +Author: Martijn van Groningen +Date: Tue Oct 22 07:41:26 2019 +0200 + + Fail with a better error when if there are no ingest nodes (#48272) + + when executing enrich execute policy api. + +commit 0ec0ab64c9cd0bdf94cd11750ce1ffb61339368d +Author: Martijn van Groningen +Date: Tue Oct 22 07:41:11 2019 +0200 + + Fix executing enrich policies stats (#48132) + + The enrich stats api picked the wrong task to be displayed + in the executing stats section. + + In case `wait_for_completion` was set to `false` then no task + was being displayed and if that param was set to `true` then + the wrong task was being displayed (transport action task instead + of enrich policy executor task). + + Testing executing policies in enrich stats api is tricky. + I have verified locally that this commit fixes the bug. + +commit c09b62d5bf3eacf4326d9f5019d3d116f2140c84 +Author: Martijn van Groningen +Date: Tue Oct 22 07:38:16 2019 +0200 + + Backport: also validate source index at put enrich policy time (#48311) + + Backport of: #48254 + + This changes tests to create a valid + source index prior to creating the enrich policy. + +commit d0a4bad95be605871291aa120451930ee316e8cb +Author: Nhat Nguyen +Date: Mon Oct 21 23:19:23 2019 -0400 + + Use MultiFileTransfer in CCR remote recovery (#44514) + + Relates #44468 + +commit 547e399dbfb493ab4b813ec528b969ed42207ab7 +Author: Tim Brooks +Date: Mon Oct 21 19:15:50 2019 -0600 + + Remove option to enable direct buffer pooling (#48310) + + This commit removes the option to change the netty system properties to + reenable the direct buffer pooling. It also removes the need for us to + disable the buffer pooling in the system properties file. Instead, we + programmatically craete an allocator that is used by our networking + layer. + + This commit does introduce an Elasticsearch property which allows the + user to fallback on the netty default allocator. If they choose this + option, they can configure the default allocator how they wish using the + standard netty properties. + +commit 0d12ef8958bb5fd1bab4eca665621f2cf96d5d7e +Author: James Baiera +Date: Mon Oct 21 16:40:49 2019 -0400 + + Add Enrich Origin (#48098) (#48312) + + This PR adds an origin for the Enrich feature, and modifies the background + maintenance task to use the origin when executing client operations. + Without this fix, the maintenance task fails to execute when security is + enabled. + +commit 2db2b945ec9193d68459c388d323b0492e03634d +Author: Przemysław Witek +Date: Mon Oct 21 22:07:19 2019 +0200 + + [7.x] Change format of MulticlassConfusionMatrix result to be more self-explanatory (#48174) (#48294) + +commit 178204703a33387139bce834a7c592292a40553e +Author: Mark Vieira +Date: Mon Oct 21 09:44:41 2019 -0700 + + Add 'javadoc' task to lifecycle check tasks (#48214) + +commit f4fa61b2f2269b23de02b293f088b316b34ca4ca +Author: James Rodewig +Date: Mon Oct 21 12:13:44 2019 -0400 + + [DOCS] Add 'Selecting gateway and seed nodes' section to CCS docs (#48297) + +commit e65c60915a2ce870162af65747a652706f8b2043 +Author: Armin Braun +Date: Mon Oct 21 17:30:35 2019 +0200 + + Cleanup FileRestoreContext Abstractions (#48173) (#48300) + + This class is only used by the blob store repository + and CCR and the abstractions didn't really make sense + with CCR ignoring the concrete `restoreFiles` method + completely and having a method used only by the blobstore + overriden as unsupported. + => Moved to a more fitting set of abstractions + => Dried up the stream wrapping in `BlobStoreRepository` a little + now that the `restoreFile` method could be simplified + + Relates #48110 as it makes changing the API of `FileRestoreContext` + to what is needed for async restores simpler + +commit cc0c876a8d03ade69714de13df241e6cb6458d86 +Author: Lee Hinman +Date: Mon Oct 21 09:14:44 2019 -0600 + + fix incorrect comparison (#48208) (#48303) + + * remove comparison of identical values + + the comparison `tookInMillis == tookInMillis` is always true. + + * add comparison between tookInMillis + +commit dc08feadc663368a164e5721e9e07a7013db47c5 +Author: Armin Braun +Date: Mon Oct 21 16:20:45 2019 +0200 + + Remove Redundant Version Param from Repository APIs (#48231) (#48298) + + This parameter isn't used by any implementation + +commit cdfac9dfb0359f5d60e48e22128450979b289f27 +Author: Alpar Torok +Date: Mon Oct 21 17:09:12 2019 +0300 + + Don't build packages on non Linux (#48246) + + * Don't build packages on non Linux + + Closes #47007 + + * Explicitly exclude windows only + +commit 5729c04df26d91d6eee41dce6a90163bb8c0ae5b +Author: Alpar Torok +Date: Mon Oct 21 17:03:29 2019 +0300 + + Fix link to GCP upload in build scans (#48248) + + * Fix link to GCP upload in build scans + + Also changed the name of the link to make it less confusing + + * Change the name + +commit a66bb2c7edb34cb287839a9ec3a3bfdb37dd9dbf +Author: James Rodewig +Date: Mon Oct 21 09:43:59 2019 -0400 + + [DOCS] Reformat CJK bigram and CJK width token filter docs (#48210) + +commit 2751a4ff1b3e115cb06e3d94e153ad06fe9cbe29 +Author: Marios Trivyzas +Date: Mon Oct 21 15:20:33 2019 +0200 + + [Docs] Fix testing docs regarding --debug-jvm (#48293) + + Remove extra whitespace and highlight the starting port number. + + Follows: #48188 + (cherry picked from commit b84b7e13c888c9a4b110c436b4eec1ffd0aebd52) + +commit abd1b5118f708abbe5cb50b6def6f00665045635 +Author: Benjamin Trent +Date: Mon Oct 21 09:21:06 2019 -0400 + + [ML] fixing tests (#48084) (#48253) + + * [ML] fixing tests + + * unmuting tests + + * reverting outlier detection job changes + +commit 0094bd59395c47a228d8e18167355821b54389f3 +Author: Tanguy Leroux +Date: Mon Oct 21 14:31:58 2019 +0200 + + Fix AutoFollowIT.testPauseAndResumeWithMultipleAutoFollowPatterns (#48289) + + The test testPauseAndResumeWithMultipleAutoFollowPatterns + failed multiple times, mostly because it creates too many leader + indices and the following cluster cannot cope with cluster state + updates generated by following indices creation and pause/ + resume auto-followers changes. + + This commit simplifies the test by creating at most 20 leader + indices and by waiting for any new leader index to be picked + up by the auto-follower before created another leader index. + It also pause and resume less auto-followers as previously. + + closes #47917 + +commit 7d085ffbd9623220cf3bfe9727ff4bb0fa02cb53 +Author: Alpar Torok +Date: Mon Oct 21 15:06:03 2019 +0300 + + Switch to debug with server=n (#48188) + + Before this change one needed to re-start debugging several times, as we + launched multiple JVMs in debug mode. + With this option the IDE has the option to re-launch and listen for + connections again leading for to a more pleasant experience. + +commit 1a42e37070d6b44e6ac9b2b1b4bc0e12f7627ac6 +Author: Przemysław Witek +Date: Mon Oct 21 13:18:08 2019 +0200 + + [7.x] Default "prediction_field_name" to (dependent_variable + "_prediction") (#48232) (#48279) + +commit 69fc715bc35764509e03a525d81ffa2fb355c29f +Author: Albert Zaharovits +Date: Mon Oct 21 13:15:05 2019 +0300 + + Fix security origin for TokenService#findActiveTokensFor... (#47418) (#48280) + + All internal searches (triggered by APIs) across the .security index + must be performed while "under the security origin". Otherwise, + the search is performed in the context of the caller which most + likely does not have privileges to search .security (hopefully). + This commit fixes this in the case of two methods in the + TokenService and corrects an overly done such context switch + in the ApiKeyService. + + In addition, this makes all tests from the client/rest-high-level + module execute as an all mighty administrator, + but not a literal superuser. + + Closes #47151 + +commit fe265f0308ebbe4044dc2c36f44df2bd36713449 +Author: Alpar Torok +Date: Fri Oct 18 08:51:37 2019 +0300 + + Mute CliSecurityIT + + tracking in #48117 + +commit 672b2a92ca5e38c6c80b35d8f208497714b959a5 +Author: David Turner +Date: Mon Oct 21 08:54:04 2019 +0100 + + Fix compile error from previous commit (#48230) + + The previous commit, 3a6fa0bbdb16348adc478ddc10894292c8be37f0 introduces a + compile error that was fixed locally but not committed. This commit adds the + missing change. + +commit 3a6fa0bbdb16348adc478ddc10894292c8be37f0 +Author: David Turner +Date: Mon Oct 21 09:43:31 2019 +0200 + + Close query cache on index service creation failure (#48230) + + Today it is possible that we create the `QueryCache` and then fail to create + the owning `IndexService` and this means we do not close the `QueryCache` + again. This commit addresses that leak. + + Fixes #48186 + +commit c903ac2376902caa6f1c05d4e58a3c8e93c0c708 +Author: Alpar Torok +Date: Mon Oct 21 10:03:44 2019 +0300 + + Fix download of 6.x rpm and deb packages (#48228) + +commit b1224fca8c74d7008e94ad505a52d5c6731cdebe +Author: Ignacio Vera +Date: Mon Oct 21 08:21:09 2019 +0200 + + upgrade to Lucene-8.3.0-snapshot-25968e3b75e (#48227) + +commit 4b3a672a0bf252d5e095e76ca1d73535039de42c +Author: Hendrik Muhs +Date: Mon Oct 21 08:19:58 2019 +0200 + + blacklist transform tests in mixed cluster prior 7.4 (#48247) + + blacklist transform tests in mixed cluster prior 7.4, see #48019 + + fixes #47943 + +commit a2217f4a915c340c857a23fd64449befd2ed1b32 +Author: Gordon Brown +Date: Fri Oct 18 14:25:18 2019 -0600 + + Fix testRetentionWhileSnapshotInProgress (#48219) + + This test could fail for two reasons, both should be fixed by this PR: + + 1) It hit a timeout for an `assertBusy`. This commit increases the + timeout for that `assertBusy`. + + 2) The snapshot that was supposed to be blocked could, in fact, be + successful. This is because a previous snapshot had been successfully + been taken, and no new data had been added between the two snapshots. + This means that no new segment files needed to be written for the new + snapshot, so the block on data files was never triggered. This commit + changes two things: First, it indexes some new data before taking the + second snapshot (the one that needs to be blocked), and second, + checks to ensure that the block is actually hit before continuing + with the test. + +commit 3ea666d6949249aa69caaea89e264e15bb81552c +Author: Christoph Büscher +Date: Fri Oct 18 20:39:07 2019 +0200 + + Clarify mapping types that support ignore_malformed (#48206) + + The `ignore_malformed` setting only works on selected mapping types, otherwise + we throw an mapper_parsing_exception. We should add a list of all the mapping + types that support it, since the number of types not supporting it seems larger. + + Closes #47166 + +commit cc26e3028184c4fcb7a438ec0f6693c844012d4a +Author: Alpar Torok +Date: Fri Oct 18 21:13:45 2019 +0300 + + Increase timeout for yml tests (#48237) + + Some of these are larger than what can complete in the regular timeout. + + Closes #48212 + +commit 0e24e6fdd191e48d4e3f750f513dce5f17eee38a +Author: Mark Vieira +Date: Fri Oct 18 11:04:17 2019 -0700 + + Upgrade to build scan plugin 2.4.2 (#48220) + +commit 403728dc3d553413ec2432c0b2a49d7c82b003fa +Author: Martijn van Groningen +Date: Fri Oct 18 16:57:47 2019 +0200 + + Use correct policy type in test + +commit a56daeae2d7e07ee3af031702ccc5a060e2fa36a +Author: Takuya Kajiwara +Date: Fri Oct 18 23:35:51 2019 +0900 + + [DOCS] Fix typos in InternalEngine.java comments (#46861) + +commit 1beeb3a83e292b396273e03d5b7e95f6fa594268 +Author: Guido Lena Cota +Date: Fri Oct 18 16:30:34 2019 +0200 + + [DOCS] Fix typos in Painless execute API docs(#47518) + +commit 5cb44a414c0c22eef71cff48c529aef81e8eb011 +Author: Martijn van Groningen +Date: Fri Oct 18 16:14:57 2019 +0200 + + Fixed links in java docs for EnrichClient (#48233) + +commit 66581d815805d229d9e41f4264feeac42f742056 +Author: Alexander Reelsen +Date: Fri Oct 18 16:14:45 2019 +0200 + + update ingest-user-agent regexes.yml (#47807) + + This new regexes are from: + https://github.com/ua-parser/uap-core/blob/154eba17f5a88a416f85d507ecd7c3a3af6f3739/regexes.yaml + +commit 844825a13f3031dd68b69159ab8fdce865dcbd69 +Author: Martijn van Groningen +Date: Fri Oct 18 16:14:07 2019 +0200 + + Validate policy type when storing an enrich policy (#48126) + +commit 1d06be2f050f8788aadbc5640cdbec162dcf2d34 +Author: James Rodewig +Date: Fri Oct 18 10:20:23 2019 -0400 + + [DOCS] Note clause limit in `index.mapping.total_fields.limit` docs (#48153) + + Notes that you should increase the `indices.query.bool.max_clause_count` + setting if you increase the `index.mapping.total_fields.limit` + setting. + + Closes #46928. + +commit fbd43196a897601b6dd815e16b9d8b0589447c60 +Author: markharwood +Date: Fri Oct 18 15:02:19 2019 +0100 + + [DOCS] Add TermVectors API reference to MLT query docs (#37228) + + Adds a reference the use of the TermVectors API following an issue raised about lack of flexibility in the MLT query: https://github.com/elastic/elasticsearch/issues/35509 + +commit 876f4aafac9e872e978a4f49ee52b6545cc1229a +Author: Benjamin Trent +Date: Fri Oct 18 10:08:17 2019 -0400 + + [ML] Add logistic_regression output aggregator (#48238) (#48244) + +commit a8bcbbc38a51a94574b3626e0baa93366e1a2bfa +Author: David Turner +Date: Fri Oct 18 15:43:55 2019 +0200 + + Quieter logging from the DiskThresholdMonitor (#48115) + + Today if an Elasticsearch node reaches a disk watermark then it will repeatedly + emit logging about it, which implies that some action needs to be taken by the + administrator. This is misleading. Elasticsearch strives to keep nodes under + the high watermark, but it is normal to have a few nodes occasionally exceed + this level. Nodes may be over the low watermark for an extended period without + any ill effects. + + This commit enhances the logging emitted by the `DiskThresholdMonitor` to be + less misleading. The expected case of hitting the high watermark and + immediately relocating one or more shards that to bring the node back under the + watermark again is reduced in severity to `INFO`. Additionally, `INFO` messages + are not emitted repeatedly. + + Fixes #48038 + +commit 5e4dd0fd2e1a9946813e9b9c4c5653699f94981f +Author: rsarawgi +Date: Fri Oct 18 06:38:34 2019 -0700 + + [ML] Removing usages of ToXContentParams.INCLUDE_TYPE (#48165) + + Removing the option of ToXContentParams.INCLUDE_TYPE and replacing them with ToXContentParams.FOR_INTERNAL_STORAGE + Closes #48057 + +commit 6a97aef8ac0343b4e0677a33059c45b61d59c21a +Author: David Turner +Date: Fri Oct 18 15:40:08 2019 +0200 + + Clarify contribution instructions re. license hdrs (#48067) + + Today our instructions in `CONTRIBUTING.md` regarding license headers are a + little unclear. This commit clarifies the exact headers that are required for + contributions. + +commit a675b35680e76f8df2ebabcb3f5fc4af3c488d86 +Author: Alpar Torok +Date: Sat Oct 5 15:36:08 2019 +0300 + + Mute org.elasticsearch.xpack.sql.jdbc.JdbcConfigurationTests.testDriverConfigurationWithSSLInURL + + tracked in #41557 + +commit ce3a06292b9e8b1d36398c29559f4b7aa8a38d7b +Author: Ioannis Kakavas +Date: Sun Oct 13 21:05:36 2019 +0300 + + Mute flaky testCreateApiKey test (#47973) + + see #47958 + +commit 11577750747a4b302cc95f9f6d1b5fe3157b91a9 +Author: Armin Braun +Date: Fri Oct 18 09:45:07 2019 +0200 + + Remove Support for pre-5.x Indices in Restore (#48181) (#48199) + + The logic for handling empty segment files has been + unnecessary ever since #24021 which removes the support + for these files in 6.x -> we can safely remove the + support for restoring these from 7.x+ to simplify the code. + +commit ca99014d8bc4e4aa40f5a439c28d58be6acb5de8 +Author: Alpar Torok +Date: Fri Oct 18 08:14:46 2019 +0300 + + Create an upload report once the build completes (#47642) + + * Create an upload report once the build completes + + We used to have this logic in Jenkins, but that forced us to make it + platform dependent and gave us less control on what to include here. + + With this change we create a single archive to be uploaded after the + build completes, and we include a link in the build scan to where we + know this should get uploaded. + + * Fix when there's nothign to upload + * Log the directory size + * Switch to ant to walk the project tree + * Collect journlas + * Filter for regular files + * only call journalctl on unix where we have bash + * Grab only logs fro this gradle version + * restrict demon log to relevant one + +commit 02d18f5c1ead0a386e5a7a20d8e2927e28754a10 +Author: Przemyslaw Gomulka +Date: Thu Oct 17 20:04:57 2019 +0200 + + [7.x] Slow log must use separate underlying logger for each index BACKPORT(#47234) (#48176) + + * Slow log must use separate underlying logger for each index (#47234) + + SlowLog instances should not share the same underlying logger, as it would cause different indexes override each other levels. When creating underlying logger, unique per index identifier should be used. Name + IndexSettings.UUID + + Closes #42432 + +commit 04e33164089bda5304150e721a3e126bdf4bd32a +Author: Armin Braun +Date: Thu Oct 17 19:27:49 2019 +0200 + + Stop Resolving Fallback IndexId (#48141) (#48204) + + There is no reason to still resolve the + fallback `IndexId` here. It only applies to + `2.x` repos and those we can't read anymore + anyway because they use an `/index` instead of + an `/index-N` blob at the repo root for which + at least 7.x+ does not contain the logic to find + it. + +commit 28f68fa22112809700f5caf70356ef7d1f692ded +Author: Przemysław Witek +Date: Thu Oct 17 18:43:15 2019 +0200 + + Make num_top_classes parameter's default value equal to 2 (#48119) (#48201) + +commit e0489fc328ec4223c55d74bcdc78c1ecd18ae3cf +Author: Dimitris Athanasiou +Date: Thu Oct 17 17:20:19 2019 +0100 + + [7.x][ML] Always refresh dest index before starting analytics process (#48090) (#48196) + + If a job stops right after reindexing is finished but before + we refresh the destination index, we don't refresh at all. + If the job is started again right after, it jumps into the analyzing state. + However, the data is still not searchable. + This is why we were seeing test failures that we start the process + expecting X rows (where X is lower than the expected number of docs) + and we end up getting X+. + + We fix this by moving the refresh of the dest index right before + we start the process so it always ensures the data is searchable. + + Closes #47612 + + Backport of #48090 + +commit eb7969e8cc6c724980ee0c7311b01544a173c91e +Author: Gordon Brown +Date: Thu Oct 17 09:59:40 2019 -0600 + + Fix ILM HLRC Javadoc->Documentation links (#48083) + + Several links from the ILM HLRC Javadoc to the online documentation were + not updated when the ILM HLRC documentation was written. This commit + fixes those links. + +commit 356eef00c8e65d957fe0a31f2ce7f6a69fe5e149 +Author: Stuart Tettemer +Date: Thu Oct 17 09:08:55 2019 -0600 + + Scripting: get context names REST API (#48026) (#48168) + + Adds `GET /_script_context`, returning a `contexts` object with each + available context as a key whose value is an empty object. eg. + ``` + { + "contexts": { + "aggregation_selector": {}, + "aggs": {}, + "aggs_combine": {}, + ... + } + } + ``` + + refs: #47411 + +commit f46281a2511dbe55fb0aef0f23b15d0ed92cbebb +Author: William Brafford +Date: Thu Oct 17 10:54:12 2019 -0400 + + Muting test that fails on Windows (#48190) + + This test has been failing very frequently on Windows checks for 7.4. We've also seen it for 7.x as well as for the new 7.5 tests. I'm muting during investigation so we can see if this is masking any other issues. + +commit befc44c1451ec2d409b1553432307ac8f90fe8c1 +Author: Mark Vieira +Date: Thu Oct 17 07:30:05 2019 -0700 + + Cache all rest tests tasks so long as they don't use shared clusters (#48161) + +commit 4c6d56bef0179043ba5f7a0a70d791bf7e9ef6c3 +Author: James Rodewig +Date: Thu Oct 17 09:30:46 2019 -0400 + + [DOCS] Expand put mapping API docs examples (#47462) + +commit b4bf47cce4fa876c87cd9d3b517c981e77a2c769 +Author: James Rodewig +Date: Thu Oct 17 09:17:58 2019 -0400 + + [DOCS] Document ingest pipeline and processor node stats (#47884) + +commit b54d817f1d4df131e510dcea79cfb646e1b76201 +Author: Anton <33043735+siffash@users.noreply.github.com> +Date: Thu Oct 17 16:14:15 2019 +0300 + + [DOCS] Fix typo in intervals query docs (#48180) + +commit 06f54428c248119f759abb0fcd8d1c17f62fa584 +Author: Hendrik Muhs +Date: Thu Oct 17 14:22:36 2019 +0200 + + update internal index name in documentation warning (#48125) + +commit 13df6beb53304fcc9ed73c3df7676fd92ff086be +Author: Alpar Torok +Date: Thu Oct 17 14:28:43 2019 +0300 + + Fix packaging tests on debian 10 (#48138) + + * Fix packaging tests on debian 10 + + - lintian no longer has `--fail-on-warnings` + - Systemd has a build number / version after it's acutal version + ``` + alpar@alpar-ci-debian-10-1015-1517:~$ systemd --version + systemd 241 (241) + +PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid + ``` + +commit ecd62934f6f628562e531d61d229fb8332a1dfa0 +Author: István Zoltán Szabó +Date: Thu Oct 17 10:22:38 2019 +0200 + + [DOCS] Applies rename of transform related roles and privileges (#48123) + +commit a5fe69c344c49c094b859b17eb9c50154c910225 +Author: Martijn van Groningen +Date: Thu Oct 17 09:51:32 2019 +0200 + + Include enrich into the info api as feature (#48157) + + This commit also fixes a bug, the enrich enabled setting + was not included in the list of settings. + + Backport of #48109 + +commit fa50377dc6f327b160c2339cb4a00f77a30fc0bf +Author: Mark Vieira +Date: Thu Oct 17 00:05:40 2019 -0700 + + Explicitly declare test fixture usage (#48166) + +commit 9bf8e1e0608a177fe0ec0e3d8b6930ae1dfd08bd +Author: Armin Braun +Date: Thu Oct 17 08:55:56 2019 +0200 + + Fix SLMSnapshotBlockingIntegTest (#47941) (#47963) + + The after snapshot action is interfering with SLM deleting snapshots + here it seems, causing concurrent delete exceptions. + Since these tests are now test-scoped there is no reason to run + snapshot deletes after each test so we can remove them to avoid this issue. + + Closes #47937 + +commit 0ca7cc18487525344161351efa2f1274d13a15aa +Author: Armin Braun +Date: Thu Oct 17 07:55:05 2019 +0200 + + Safely Close Repositories on Node Shutdown (#48020) (#48107) + + We were not closing repositories on Node shutdown. + In production, this has little effect but in tests + shutting down a node using `MockRepository` and is + currently stuck in a simulated blocked-IO situation + will only unblock when the node's threadpool is + interrupted. This might in some edge cases (many + snapshot threads and some CI slowness) result + in the execution taking longer than 5s to release + all the shard stores and thus we fail the assertion + about unreleased shard stores in the internal test cluster. + + Regardless of tests, I think we should close repositories + and release resources associated with them when closing + a node and not just when removing a repository from the CS + with running nodes as this behavior is really unexpected. + + Fixes #47689 + +commit f1bc3a0753449f5effe16eba8b2a65df417bfb68 +Author: Armin Braun +Date: Thu Oct 17 07:54:20 2019 +0200 + + Remove TestLogging for #46701 (#48156) (#48160) + + This hasn't failed in 5 weeks now. Removing + the test logging and closing the issue. + + Closes #46701 + +commit 5af66d79ef32a15aa64b0bd16edf5f8e0810e087 +Author: Lee Hinman +Date: Wed Oct 16 21:06:27 2019 -0600 + + Add SLM support to xpack usage and info APIs (#48149) + + * Add SLM support to xpack usage and info APIs + + This is a backport of #48096 + + This adds the missing xpack usage and info information into the + `/_xpack` and `/_xpack/usage` APIs. The output now looks like: + + ``` + GET /_xpack/usage + { + ... + "slm" : { + "available" : true, + "enabled" : true, + "policy_count" : 1, + "policy_stats" : { + "retention_runs" : 0, + ... + } + } + ``` + + and + + ``` + GET /_xpack + { + ... + "features" : { + ... + "slm" : { + "available" : true, + "enabled" : true + }, + ... + } + } + ``` + + Relates to #43663 + + * Fix missing license + +commit fa99721295d76e5b297c99e9032b3dfb0aaa5649 +Author: Jack Conradson +Date: Wed Oct 16 16:09:22 2019 -0700 + + Drop stored scripts with the old style-id (#48078) + + This PR fixes (#47593). Stored scripts with the old-style id of lang#id are + saved through the upgrade process but are no longer accessible in recent + versions. This fix will drop those scripts altogether since there is no way for + a user to access them. + +commit ac1ed6e74053d0df0e7fc2a68b804080dc618d79 +Author: William Brafford +Date: Wed Oct 16 16:24:50 2019 -0400 + + Update TESTING.asciidoc for new :qa:os test style (#48023) + + * Update testing document for new packaging tests + + The TESTING.asciidoc document had gotten out of date due to some new and + wonderful changes in our vagrant testing code. I've removed all of the + instructions that no longer work, and added working examples and descriptions + in their place. + +commit 73f5fef6783881ff0ab50601842c660c3d170135 +Author: David Woods +Date: Wed Oct 16 16:13:26 2019 -0400 + + Add a note to query_string docs around spaces in field names. (#47326) + +commit ee110c2d422beec0e51ebd7274e0f6952bd62759 +Author: Benjamin Trent +Date: Wed Oct 16 15:46:50 2019 -0400 + + [ML] Muting tests due to #48085 (#48086) (#48154) + +commit 0dddbb5b426ca4edfe5b59644863ab98713fb411 +Author: Benjamin Trent +Date: Wed Oct 16 15:46:20 2019 -0400 + + [ML] Parse and index inference model (#48016) (#48152) + + This adds parsing an inference model as a possible + result of the analytics process. When we do parse such a model + we persist a `TrainedModelConfig` into the inference index + that contains additional metadata derived from the running job. + +commit 74812f78dd4c190ad9a5c4931997e960cc42fdd7 +Author: Michael Basnight +Date: Wed Oct 16 12:54:33 2019 -0500 + + Add enrich_user to security docs (#48079) + +commit 4c04006dede8b0ce9232e14c4d2fd806c6e29e33 +Author: Mark Vieira +Date: Wed Oct 16 10:30:38 2019 -0700 + + Create test superuser when launching elasticsearch via 'run' task (#48072) + + + (cherry picked from commit 225d1c9ac3cd869e61e52717fab9f613883d276e) + +commit 2d98fec2477ab55f0bc0134e8fc8272f17bf0620 +Author: Mark Vieira +Date: Fri Oct 11 15:37:35 2019 -0700 + + Add build scan tag for all pull request builds (#47889) + +commit b2dc98562b7261ee2477209470ab4ea20082f045 +Author: jimczi +Date: Wed Oct 16 15:44:21 2019 +0200 + + Bump version to 7.6 + +commit 8677653c5bd8d3bab716d6fd3bbf9ad58fbd0aac +Author: James Rodewig +Date: Wed Oct 16 08:50:12 2019 -0400 + + [DOCS] Reformat apostrophe token filter docs (#48076) + +commit f501a4b2b5d090761953287a9bb09826a89d1721 +Author: François-Clément Brossard +Date: Wed Oct 16 20:22:54 2019 +0900 + + Clarify low watermark documentation (#48112) + + Today the docs say that the low watermark has no effect on any shards that have + never been allocated, but this is confusing. Here "shard" means "replication + group" not "shard copy" but this conflicts with the "never been allocated" + qualifier since one allocates shard copies and not replication groups. + + This commit removes the misleading words. A newly-created replication group + remains newly-created until one of its copies is assigned, which might be quite + some time later, but it seems better to leave this implicit. + +commit 3233bce8cb1680cf090f12b59cc3d16b08d8bd43 +Author: Marios Trivyzas +Date: Wed Oct 16 12:45:57 2019 +0200 + + SQL: Fix issue with negative literels and parentheses (#48113) + + Previously when a numeric literal was enclosed in parentheses and then + negated, the negation was lost and the number was considered positive, e.g.: + `-(5)` was considered as `5` instead of `-5` + `- ( (1.28) )` was considered as `1.28` instead of `-1.28` + + Fixes: #48009 + + (cherry picked from commit 4dee4bf3b34081062ba2e28ab8524a066812a180) + +commit 8f815240b3f870355e59e8a3ff96d5608fd98378 +Author: Przemysław Witek +Date: Wed Oct 16 11:09:56 2019 +0200 + + [7.x] Allow integer types for classification's dependent variable (#47902) (#48080) + +commit 8243e99134614dbef09611d7c775e3c77ab8ab79 +Author: Klemen Košir +Date: Wed Oct 16 06:04:08 2019 +0900 + + Fix typo in QueryBuilders Javadoc. (#47362) + + This PR fixes a typo in the Javadoc for terms queries in QueryBuilders. + +commit 28c1677341195e03bb405113d0b1173f663c6204 +Author: Ian Danforth <52720125+iandanforth-alation@users.noreply.github.com> +Date: Tue Oct 15 12:56:16 2019 -0700 + + [DOCS] Fix typo in percentile rank aggregation docs (#47247) + +commit 8c73e215b24c8c03a9fa682027d24d6d4af1f033 +Author: Wilder Pereira +Date: Tue Oct 15 16:52:52 2019 -0300 + + [DOCS] Remove unneeded spaces from custom analyzer snippet (#47332) + +commit 09604dbaeaad100c72917b2b84bb7c1c62fa672e +Author: Alex Pang +Date: Tue Oct 15 15:50:32 2019 -0400 + + [DOCS] Fix truststores typo (#47738) + +commit 601a88bede39a74b9023886ea38e2b8029e1555c +Author: James Rodewig +Date: Tue Oct 15 15:46:50 2019 -0400 + + [DOCS] Sort analyzers, tokenizers, and token filters alphabetically (#48068) + +commit d9c7e3847eef6b1358a1546c8229a4053d7a2ce8 +Author: David Roberts +Date: Tue Oct 15 19:58:48 2019 +0100 + + [TEST] Don't assert order of data frame analytics audit messages (#48065) + + Audit messages are stored with millisecond timestamps. If two + messages have the same millisecond timestamp then asserting on + their order is impossible given the information available. + + This PR changes the assertion on audit messages in the native + data frame analytics tests to assert that the expected audit + messages exist in any order. + + Fixes #48035 + +commit 3a7c2a4d1718ad49f1c3a9af0f86c447294e949e +Author: James Rodewig +Date: Tue Oct 15 13:46:55 2019 -0400 + + [DOCS] Add `wait_for_completion` parm to execute enrich policy API docs (#48077) + +commit eaa56344b56afd58464df416bb7e02bb57580104 +Author: Przemysław Witek +Date: Tue Oct 15 18:33:20 2019 +0200 + + Verify that the failure reason of analytics process is empty (#48042) (#48071) + +commit aff0c9babccd98f2ed4624f05644cce1b2ea54a2 +Merge: 05b3de01108 31e41d4ac2f +Author: Martijn van Groningen +Date: Tue Oct 15 17:31:45 2019 +0200 + + This commits merges (#48040) the enrich-7.x feature branch, + which is backport merge and adds a new ingest processor, named enrich processor, + that allows document being ingested to be enriched with data from other indices. + + Besides a new enrich processor, this PR adds several APIs to manage an enrich policy. + An enrich policy is in charge of making the data from other indices available to the enrich processor in an efficient manner. + + Related to #32789 + +commit 05b3de01108233364cd13f0b0575833427a1a7f8 +Author: James Rodewig +Date: Tue Oct 15 11:16:20 2019 -0400 + + [DOCS] Expand `updated` response parm in reindex API docs (#48008) + +commit 155ecd0a76b3bd32c2896eda5f327b14a3c52ec1 +Author: Jack Conradson +Date: Tue Oct 15 07:46:29 2019 -0700 + + Change Painless regex node to use SField instead of Globals (#47944) + + * Change Painless regex node to use SField instead of Globals + + * Use reflection instead of ASM to specify modifiers + + * Remove synthetic from SField + +commit b2ce72850b563ed1b644e95a60af3976de6e20a6 +Author: Hendrik Muhs +Date: Tue Oct 15 16:14:39 2019 +0200 + + [7.5][Transform] prevent assignment if any node is older than 7.4 (#48055) + + disable task assignment of transforms if any node uses version 7.2 or 7.2 (mixed cluster). + + fixes #48019 + +commit f0cb43fb96aea2b3608789b02b0e9fe9360babfc +Author: Uwe Schindler +Date: Tue Oct 15 15:19:41 2019 +0200 + + Update forbiddenapis to v2.7 (#47969) + +commit 6c9305dc789af6d150582e9e526a0557ccc0c2fe +Author: Alpar Torok +Date: Tue Oct 15 16:43:46 2019 +0300 + + Partial Revert "Convert RunTask to use testclusers, remove ClusterFormationTasks (#47572)" + + This reverts the removal of the ClusterFormationTaks from + commit 36d018c90907d1f7da24bd8449248e81b17c2530 so they are usable for a + bit longer in the hadoop build. + +commit 7fddf198b74ecd68421be1babd999e33946277a5 +Author: Marios Trivyzas +Date: Tue Oct 15 14:42:20 2019 +0200 + + SQL: Implement DATEDIFF function (#47920) + + Implement DATEDIFF/TIMESTAMPDIFF function as per the MS-SQL spec: + https://docs.microsoft.com/en-us/sql/t-sql/functions/datediff-transact-sql?view=sql-server-2017 + which allows a user to substract two date/datetime fields and return the + difference in the date/time unit specified. + + Closes: #47919 + (cherry picked from commit 745699f38dc8222670ffd65b66df33b5da39040b) + +commit 4aa7c7bad66bf3f794e74ecae2384c1aa3926523 +Author: Hendrik Muhs +Date: Tue Oct 15 15:04:09 2019 +0200 + + [Transform] add alias for backwards compatibility with 7.4 (#48049) + + add alias for backwards compatibility with 7.4 + + relates #47943 + +commit 620bd9d2241c4dad2a687f459e0e6230b119cf92 +Author: Przemysław Witek +Date: Tue Oct 15 14:49:16 2019 +0200 + + Enable test testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows_TopClassesRequested now that top classes are correctly reported by C++. (#48043) (#48053) + +commit a61d12afea5464035df502f0d9dd79dd2df91b82 +Author: James Rodewig +Date: Tue Oct 15 08:47:34 2019 -0400 + + [DOCS] Add #46860 to 7.4 release notes (#48018) + +commit 361e7ad0ef2cbae42bcbdc30cd12700e42a76a53 +Author: Benjamin Trent +Date: Tue Oct 15 07:52:13 2019 -0400 + + [ML][Transforms] fix bwc serialization with 7.3 (#48021) (#48048) + +commit 83321b0e5e6ad0a4d9f3bdee0dcbcbaab99396aa +Author: David Roberts +Date: Tue Oct 15 12:17:32 2019 +0100 + + [ML] Fix isNoop() for datafeed update (#48046) + + max_empty_searches = -1 in a datafeed update implies + max_empty_searches will be unset on the datafeed when + the update is applied. The isNoop() method needs to + take this -1 to null equivalence into account. + +commit 6589617a5182bbc0ffb6dc1f31778013089b9b74 +Author: Marios Trivyzas +Date: Tue Oct 15 12:25:04 2019 +0200 + + SQL: Fix arg verification for DateAddProcessor (#48041) + + Previously, the safety check for the 2nd argument of the DateAddProcessor was + restricting it to Integer which was wrong since we allow all non-rational + numbers, so it's changed to a Number check as it's done in other cases. + + Enhanced some tests regarding the check for an integer (non-rational + argument). + + (cherry picked from commit 0516b6eaf5eb98fa5bd087c3fece80139a6b118e) + +commit 5caa10134545e5541328919b3f70ca86d0b9095d +Author: Armin Braun +Date: Tue Oct 15 12:10:19 2019 +0200 + + Fix Bug in Azure Repo Exception Handling (#47968) (#48030) + + We were incorrectly handling `IOExceptions` thrown by + the `InputStream` side of the upload operation, resulting + in a `ClassCastException` as we expected to never get + `IOException` from the Azure SDK code but we do in practice. + This PR also sets an assertion on `markSupported` for the + streams used by the SDK as adding the test for this scenario + revealed that the SDK client would retry uploads for + non-mark-supporting streams on `IOException` in the `InputStream`. + +commit 31e41d4ac2fb83942b3601a72fcf7f1ee9386b0f +Author: Martijn van Groningen +Date: Tue Oct 15 10:29:47 2019 +0200 + + fixed invalid reference + +commit b858e19bcc239fb2630345cb5e8022c4e96d9ca1 +Author: jimczi +Date: Tue Oct 15 09:40:59 2019 +0200 + + Revert #46598 that breaks the cachability of the sub search contexts. + +commit fe15d9f784f86a974f44cdf6fa84324ccb8a62bf +Author: Tanguy Leroux +Date: Tue Oct 15 08:38:49 2019 +0200 + + Add documentation for Pause/Resume Auto-Follower APIs (#47985) (#48034) + + Relates #47510 + + Co-Authored-By: James Rodewig + +commit 984323783e4c876f0f416e4680b6b61dc4299087 +Author: David Roberts +Date: Tue Oct 15 06:55:11 2019 +0100 + + [ML][7.x] Add lazy assignment job config option (#47993) + + This change adds: + + - A new option, allow_lazy_open, to anomaly detection jobs + - A new option, allow_lazy_start, to data frame analytics jobs + + Both work in the same way: they allow a job to be + opened/started even if no ML node exists that can + accommodate the job immediately. In this situation + the job waits in the opening/starting state until ML + node capacity is available. (The starting state for data + frame analytics jobs is new in this change.) + + Additionally, the ML nightly maintenance tasks now + creates audit warnings for ML jobs that are unassigned. + This means that jobs that cannot be assigned to an ML + node for a very long time will show a yellow warning + triangle in the UI. + + A final change is that it is now possible to close a job + that is not assigned to a node without using force. + This is because previously jobs that were open but + not assigned to a node were an aberration, whereas + after this change they'll be relatively common. + +commit 77164e9017e52e7109df49ce44f712497c9599f4 +Author: Martijn van Groningen +Date: Tue Oct 15 07:41:41 2019 +0200 + + adjusted minimal supported version + +commit cc4b6c43b3ded1ca90a40ef3fa7bdc6fb46bf0ed +Merge: 51c33f3edfe 300ddfa3c19 +Author: Martijn van Groningen +Date: Tue Oct 15 07:23:47 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 51c33f3edfe0db1a11d659d30e2a499e19f9ddca +Author: Martijn van Groningen +Date: Tue Oct 15 07:16:49 2019 +0200 + + remove eclipse conditional + +commit c4b1a3045ade262c840f2d2a8f4699212ba98f5a +Author: Martijn van Groningen +Date: Tue Oct 15 07:02:11 2019 +0200 + + Fixed test, take into account that Map can be the result if max_matches is 1. + +commit 300ddfa3c19bcf625c9ef45a86e0cf8dbae1c81c +Author: Gordon Brown +Date: Mon Oct 14 16:56:31 2019 -0600 + + SLM Start/Stop HLRC and docs (#47966) + + This commit adds HLRC support and documentation for the SLM Start and + Stop APIs, as well as updating existing documentation where appropriate. + + This commit also ensures that the SLM APIs are properly included in the + HLRC documentation. + +commit 18d7e32b7d0f9eccfb2fbecd55298acaf64e207e +Author: James Baiera +Date: Mon Oct 14 15:38:07 2019 -0400 + + Add wait for completion for Enrich policy execution (#47886) + + This PR adds the ability to run the enrich policy execution task in the background, + returning a task id instead of waiting for the completed operation. + +commit 8814bf07f1dd391828e0ddb3fcf5f95dfc5b4a2e +Author: Tim Brooks +Date: Mon Oct 14 13:39:08 2019 -0600 + + Upgrade to Netty 4.1.42 (#48015) + + Upgrades the netty version. + +commit 7fc9198d46dd3316304a76536adac525c8716784 +Author: Martijn van Groningen +Date: Mon Oct 14 21:04:47 2019 +0200 + + Change how `max_matches` affects `target_field` option. (#47982) + + Prior to this change the `target_field` would always be a json array + field in the document being ingested. This to take into account that + multiple enrich documents could be inserted into the `target_field`. + + However the default `max_matches` is `1`. Meaning that by default + only a single enrich document would be added to `target_field` json + array field. + + This commit changes this; if `max_matches` is set to `1` then the single + document would be added as a json object to the `target_field` and + if it is configured to a higher value then the enrich documents will be + added as a json array (even if a single enrich document happens to be + enriched). + +commit 5a4745ae69a342582b51a8801b443d529862b0fd +Author: Jake Landis +Date: Mon Oct 14 13:40:28 2019 -0500 + + Re-enable Watcher full cluster restart test (#47950) (#48000) + + This test is believed to be fixed by #43939 + + closes #40178 + +commit 17d8ee9a9ce1cfc495fa73f273397cd1e9db39b3 +Author: Hendrik Muhs +Date: Mon Oct 14 20:11:14 2019 +0200 + + [Transform] wait for deprecated index shards to get active (#47997) + + wait for deprecated index shards to get active + +commit 7cc73f61938a18d0044c1ada15b143895d5f8c32 +Author: Martijn van Groningen +Date: Mon Oct 14 19:44:47 2019 +0200 + + Add HLRC support for enrich execute policy API (#47991) + + This PR also includes HLRC docs for the enrich stats api. + + Relates to #32789 + +commit f6f5efe141ef9626d324a76655269531bceb0149 +Author: Michael Basnight +Date: Mon Oct 14 12:46:16 2019 -0500 + + Add cloudId builder to the HLRC (#47868) + + Elastic cloud has a concept of a cloud Id. This Id is a base64 encoded + url, split up into a few parts. This commit allows the user to pass in a + cloud id now, which is translated to a HttpHost that is defined by the + encoded parts therein. + +commit 170266765b9a5c2fdf30188fa3998904e5711477 +Author: James Rodewig +Date: Mon Oct 14 12:29:11 2019 -0400 + + [DOCS] Reformat docs for several snapshot lifecycle policy APIs (#47998) + +commit e4ea8b46b60e5e4dc7715d781ee0b35a5cf436b7 +Author: Tanguy Leroux +Date: Mon Oct 14 18:25:53 2019 +0200 + + Add Pause/Resume Auto-Follower APIs to High Level REST Client (#48004) + + This commit adds support for Pause/Resume Auto-Follower APIs + to the HLRC, with the documentation. + + Relates #47510 + +commit 699d4d4c6f89cc39e4979c1a0c9f7cdbfbb94881 +Author: Gordon Brown +Date: Mon Oct 14 10:19:57 2019 -0600 + + Manage retention of partial snapshots in SLM (#47833) + + Currently, partial snapshots will eventually build up unless they are + manually deleted. Partial snapshots may be useful if there is not a more + recent successful snapshot, but should eventually be deleted if they are + no longer useful. + + With this change, partial snapshots are deleted using the following + strategy: PARTIAL snapshots will be kept until the configured + expire_after period has passed, if present, and then be deleted. If + there is no configured expire_after in the retention policy, then they + will be deleted if there is at least one more recent successful snapshot + from this policy (as they may otherwise be useful for troubleshooting + purposes). Partial snapshots are not counted towards either min_count or + max_count. + +commit 1ca25bed385e16fcfa22406ecab00f794d776773 +Author: David Roberts +Date: Mon Oct 14 17:19:13 2019 +0100 + + [ML][7.x] Add option to stop datafeed that finds no data (#47995) + + Adds a new datafeed config option, max_empty_searches, + that tells a datafeed that has never found any data to stop + itself and close its associated job after a certain number + of real-time searches have returned no data. + + Backport of #47922 + +commit 5f3ef2e09ca31561b6beb6773d8d3cf3dbf96054 +Author: James Rodewig +Date: Mon Oct 14 11:38:46 2019 -0400 + + [DOCS] Remove unsupported `local` and `master_timeout` parms from cat API docs (#47933) + +commit 508db4589bc2497b144f89afa47b0b77278e713e +Author: Benjamin Trent +Date: Mon Oct 14 11:17:11 2019 -0400 + + [ML][Transforms] signal listener early on stop failure (#47954) (#48002) + +commit 2b1372adfd86998512669304e35fdf7ad76e9351 +Author: Ioannis Kakavas +Date: Mon Oct 14 17:55:46 2019 +0300 + + File based role mappings vs the role mapping APIs (#47015) (#47978) + + Make clear in the docs that the role mapping APIs is the preferred + way to manage role mappings and that the role mappings that are + defined in files cannot be viewed or managed with the APIs + +commit c2a3e834279b56174d09c15906a70f3cbf2d1fca +Author: Tanguy Leroux +Date: Mon Oct 14 16:19:34 2019 +0200 + + Remove unused transport action from TransportFreezeIndexAction (#47992) + + Removes unnecessary TransportCloseIndexAction from + TransportFreezeIndexAction + +commit f48981f43c34545cc072be9201cd6f6eeb9ac55c +Author: Martijn van Groningen +Date: Mon Oct 14 14:34:09 2019 +0200 + + Remove redundant nested operator in builtin grok expression. (#47870) + + This prevents the following warning from being printed to console: + `regular expression has redundant nested repeat operator + /%\{(?(?[A-z0-9]+)(?::(?[[:alnum:]@\[\]_:.-]+))?)(?:=(?(?:(?:[^{}]+|\.+)+)+))?\}/` + + The current grok expression is not failing, but just this warning is being printed. + The warning started being printed after upgrading joni (#47374). + + Closes #47861 + +commit ef02a736ca2de551267f1a3eb6328eec87510858 +Author: Jim Ferenczi +Date: Mon Oct 14 13:06:36 2019 +0200 + + Don't apply the plugin's reader wrapper in can_match phase (#47816) + + This change modifies the local execution of the `can_match` phase to **not** apply + the plugin's reader wrapper (if it is configured) when acquiring the searcher. + We must ensure that the phase runs quickly and since we don't know the cost + of applying the wrapper it is preferable to avoid it entirely. The can_match + phase can aford false positives so it is also safe for the builtin plugins + that use this functionality. + + Closes #46817 + +commit d4901a71d79d9e691741a1517e77d38b2bd14a3b +Merge: 73263c654ac 8180cf1e68b +Author: Martijn van Groningen +Date: Mon Oct 14 10:27:17 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 9ee7b3743e3f7cb77f0cdf15d1acfccd4ee238c3 +Author: Ioannis Kakavas +Date: Mon Oct 14 10:40:24 2019 +0300 + + Add FIPS 140 mode to XPack Usage API (#47278) (#47976) + + This change adds support for the FIPS 140 mode feature to be + retrieved via the XPack Usage API. + +commit 8180cf1e68b4bf50f7633e515b80da46bd2e3f9c +Author: Nhat Nguyen +Date: Sun Oct 13 21:19:14 2019 -0400 + + Mute testDoNotInfinitelyWaitForMapping + + Tracked at #47974 + +commit 2995d4a9c0fbc19890c3e14e0e1f8042915d1ed6 +Author: Nhat Nguyen +Date: Sun Oct 13 12:58:05 2019 -0400 + + Sequence number based replica allocation (#46959) + + With this change, shard allocation prefers allocating replicas on a node + that already has a copy of the shard that is as close as possible to the + primary, so that it is as cheap as possible to bring the new replica in + sync with the primary. Furthermore, if we find a copy that is identical + to the primary then we cancel an ongoing recovery because the new copy + which is identical to the primary needs no work to recover as a replica. + + We no longer need to perform a synced flush before performing a rolling + upgrade or full cluster start with this improvement. + + Closes #46318 + +commit 4f0622592815e4c5111a9671dbb8cf205061dc2f +Author: Nhat Nguyen +Date: Sun Oct 13 12:48:05 2019 -0400 + + Avoid unneeded refresh with concurrent realtime gets (#47895) + + This change should reduce refreshes for a use-case where we perform + multiple realtime gets at the same time on an active index. Currently, + we only call refresh if the index operation is still on the versionMap. + However, at the time we call refresh, that operation might be already or + will be included in the latest reader. Hence, we do not need to refresh. + Adding another lock here is not an issue as the refresh is already + sequential. + +commit 4c1bb210cb69c4350b1105f98c5156dda91f8817 +Author: Nhat Nguyen +Date: Fri Oct 11 21:09:54 2019 -0400 + + Force flush in translog retention policy test (#47879) + + If we roll translog but do not index, then a flush without force is a + noop. In this case, the number of retained translog files will be higher + than the value specified by the retention policy. + + Closes #4741 + +commit 46ae86ac3141487f2f60535ac6f6ef5181e2eb5a +Author: David Roberts +Date: Sun Oct 13 20:06:42 2019 +0100 + + [ML] Fix detection of syslog-like timestamp in find_file_structure (#47970) + + Usually syslog timestamps have two spaces before a single + digit day-of-month. However, in some non-syslog cases + where syslog-like timestamps are used there is only one + space. The grok pattern supports this, so the timestamp + parser should too. This change makes the + find_file_structure endpoint do this. + + Also fixes another problem that the same test case + exposed in the find_file_structure endpoint, which was + that the exclude_lines_pattern for delimited files was + always created on the assumption the delimiter was a + comma. Now it is based on the actual delimiter. + +commit 742fa818b85bb04b802c1c4d11f97b9c964b4cf1 +Author: Tanguy Leroux +Date: Sun Oct 13 09:22:51 2019 +0200 + + Add Pause/Resume Auto Follower APIs (#47510) (#47904) + + This commit adds two APIs that allow to pause and resume + CCR auto-follower patterns: + + // pause auto-follower + POST /_ccr/auto_follow/my_pattern/pause + + // resume auto-follower + POST /_ccr/auto_follow/my_pattern/resume + + The ability to pause and resume auto-follow patterns can be + useful in some situations, including the rolling upgrades of + cluster using a bi-directional cross-cluster replication scheme + (see #46665). + + This commit adds a new active flag to the AutoFollowPattern + and adapts the AutoCoordinator and AutoFollower classes so + that it stops to fetch remote's cluster state when all auto-follow + patterns associate to the remote cluster are paused. + + When an auto-follower is paused, remote indices that match the + pattern are just ignored: they are not added to the pattern's + followed indices uids list that is maintained in the local cluster + state. This way, when the auto-follow pattern is resumed the + indices created in the remote cluster in the meantime will be + picked up again and added as new following indices. Indices + created and then deleted in the remote cluster will be ignored + as they won't be seen at all by the auto-follower pattern at + resume time. + + Backport of #47510 for 7.x + +commit 65717f6f420b2e3afd0dc089ae490d4aa1d14e90 +Author: Marios Trivyzas +Date: Fri Oct 11 19:10:01 2019 +0200 + + SQL: Fix Nullability of DATEADD (#47921) + + Previously, Nullability was set to UNKNOWN instead of TRUE which + resulted on QueryFolder not correctly folding to NULL if any of the args + was null. + + Remove the overriding nullable() also for DatePart/DateTrunc to allow + delegation the parent class. + + (cherry picked from commit 05a7108e133b5ae7bec2257db5ae2d30ad926ee2) + +commit ac209c142c92a6ef67a350242ac851577faf306f +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sat Oct 12 22:22:16 2019 +1100 + + Remove uniqueness constraint for API key name and make it optional (#47549) (#47959) + + Since we cannot guarantee the uniqueness of the API key `name` this commit removes the constraint and makes this field optional. + + Closes #46646 + +commit dc8080e88afdd204b7c800acbce0892daaf7f7bd +Author: Ryan Ernst +Date: Fri Oct 11 15:22:06 2019 -0700 + + Add snapshot support to distribution download plugin (#47837) + + The distribution download plugin which handles finding built + distributions for testing currently only knows how to find locally built + snapshots. When an external Elasticsearch plugin uses build-tools, these + snapshots do not exist. This commit extends the download plugin so it + pulls from the Elastic snapshots service when used outside of the + Elasticsearch repository. + + closes #47123 + +commit 6ab58de7ef9f4d1884be842667686bdf425a3e98 +Author: Przemyslaw Gomulka +Date: Fri Oct 11 21:19:56 2019 +0200 + + [7.x] Enable ResolverStyle.STRICT for java formatters backport(#46675) (#47913) + + Joda was using ResolverStyle.STRICT when parsing. This means that date will be validated to be a correct year, year-of-month, day-of-month + However, we also want to make it works with Year-Of-Era as Joda used to, hence custom temporalquery.localdate in DateFormatters.from + Within DateFormatters we use the correct uuuu year instead of yyyy year of era + + worth noting: if yyyy(without an era) is used in code, the parsing result will be a TemporalAccessor which will fail to be converted into LocalDate. We mostly use DateFormatters.from so this takes care of this. If possible the uuuu format should be used. + +commit 627faf1850f3337b619f4d2496a16c3467558de6 +Author: Benjamin Trent +Date: Fri Oct 11 15:11:16 2019 -0400 + + [7.x] [ML][Analytics] fix bug where regression deleted early does not delete state (#47885) (#47914) + + * [ML][Analytics] fix bug where regression deleted early does not delete state (#47885) + + * [ML][Analytics] fix bug where regression deleted early does not delete state + + * Fixing ml with security test failure + + * fixing for older java + +commit 7f01b0a67020df49b5933e218093253a722c2cf7 +Author: Nick Knize +Date: Fri Oct 11 14:06:15 2019 -0500 + + Mute AzureBlobStoreRepositoryTests.testIndicesDeletedFromRepository (#47949) + +commit 68eaa21d77ae4e78059d0e1a3bbaa03b4df31d6c +Author: Nick Knize +Date: Fri Oct 11 12:41:50 2019 -0500 + + Mute testBasicFailureRetention (#47940) + +commit c57191b16398d78fdf114e3159f288ce722adec8 +Author: Chris Roberson +Date: Fri Oct 11 14:54:59 2019 -0400 + + [Monitoring] Add new cluster privilege now necessary for the stack monitoring ui (#47871) (#47915) + + * Add new cluster privilege now necessary for the stack monitoring ui + + * PR feedback, and add test + +commit 1636fa5f158f81b23efd49163c1b29659f8a8948 +Author: Benjamin Trent +Date: Fri Oct 11 14:49:20 2019 -0400 + + [ML][Transforms] Muting tests in 7.x (#47946) + +commit fe8767789a959d77647f433bd4bde5dfdcd672bf +Author: William Brafford +Date: Fri Oct 11 14:47:34 2019 -0400 + + GlobalBuildInfo plugin should search packed references for commit IDs (#47464) (#47936) + + * GlobalBuildInfo plugin searches packed references + + In recent versions of Git, references may be packed in a packed-refs + file. If this happens, Gradle will need to look in that file to find + build information. + +commit 73263c654ace5a77310d09bf64c70d22a53dd622 +Author: James Baiera +Date: Wed Oct 9 14:37:53 2019 -0400 + + Add basic task support for executing enrich policies (#47523) + + Changes the execution logic to create a new task using the execute request, + and attaches the new task to the policy runner to be updated. Also, a new + response is now returned from the execute api, which contains either the task + id of the execution, or the completed status of the run. The fields are mutually + exclusive to make it easier to discern what type of response it is. + +commit 256102042b51a3181cbb1c4c6d75f8514fbd35ae +Author: James Rodewig +Date: Fri Oct 11 13:00:48 2019 -0400 + + [DOCS] Reformat get snapshot lifecycle policy API docs (#47827) (#47930) + +commit 2ef12c37f519e8997e2e9e07c9465217e78a59f1 +Author: Christoph Büscher +Date: Fri Oct 11 18:18:40 2019 +0200 + + Add builder for distance_feature to QueryBuilders (#47846) + + The QueryBuilders convenience class is currently missing a shortcut to construct + a DistanceFeatureQueryBuilder, which is added here. + + Closes #47767 + +commit 5cbc96ccb60288a31043f28aff4132182c70a981 +Author: Alpar Torok +Date: Thu Oct 10 17:21:40 2019 +0300 + + Fix dependency info tasks (#47857) + + We fixed warnings related to task input and outputs in #45098. + This particular input was not considered, a warning was present for it + and Gradle didn't use it as part of task inputs. + As soon as we fixed it Gradle started considering it an input and + enforced that it exists. + + With this change we make it optional as the task can work both wih and + without this directory. + +commit ec9198d0e236a9794da53df63d80c3258ea5f8f1 +Author: Alan Woodward +Date: Fri Oct 11 17:01:42 2019 +0100 + + Adjust Version.V_6_8_4 to refer to Lucene 7.7.2 (#47926) + + 6.8.4 will ship with Lucene 7.7.2, so we need to change our version settings to + reflect this. + + Relates #47901 + +commit aa5689f5b309a8640dce69328171fce6a546ceeb +Author: James Rodewig +Date: Fri Oct 11 11:28:38 2019 -0400 + + [DOCS] Reformat put snapshot lifecycle policy API docs (#47811) (#47925) + +commit 0ca53bd80e9c54ad6d6823d63140841445ee637f +Author: Hendrik Muhs +Date: Fri Oct 11 17:15:01 2019 +0200 + + add BWC alias for internal index + + create an alias for old nodes to retrieve new documents in the internal index as they do not know the new index pattern + +commit ba62eb3dce438b40f85040c0f5ab3295f5fee53c +Author: David Turner +Date: Fri Oct 11 16:43:59 2019 +0200 + + Allow truncation of clean translog (#47866) + + Today the `elasticsearch-shard remove-corrupted-data` tool will only truncate a + translog it determines to be corrupt. However there may be other cases in which + it is desirable to truncate the translog, for instance if an operation in the + translog cannot be replayed for some reason other than corruption. This commit + adds a `--truncate-clean-translog` option to skip the corruption check on the + translog and blindly truncate it. + +commit 58289a857dc949155b0d9662c3ffb2f42429bb1a +Author: James Rodewig +Date: Fri Oct 11 09:40:49 2019 -0400 + + [DOCS] Reformat reload search analyzers API docs (#47824) (#47907) + +commit 33705c4b953af566b3bf77e74841ae667fa05f11 +Author: Ioannis Kakavas +Date: Fri Oct 11 16:34:11 2019 +0300 + + Document SAML APIs (#45105) (#47909) + + This change adds documentation for the SAML APIs in Elasticsearch + and adds simple instructions on how these APIs can be used to + authenticate a user with SAML by a custom web application other + than Kibana. + + Resolves: #40352 + +commit c62fe8c344582a91d2f544e1960471f79015f446 +Author: Przemysław Witek +Date: Fri Oct 11 14:57:08 2019 +0200 + + Require that the dependent variable column has at most 2 distinct values in classfication analysis. (#47858) (#47906) + +commit a0d0866f5965912ee77bc95812429abad624b11f +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Oct 11 14:22:04 2019 +0200 + + Shrink should not touch max_retries (#47719) + + Shrink would set `max_retries=1` in order to avoid retrying. This + however sticks to the shrunk index afterwards, causing issues when a + shard copy later fails to allocate just once. + + Avoiding a retry of a shrink makes sense since there is no new node + to allocate to and a retry will likely fail again. However, the downside of + having max_retries=1 afterwards outweigh the benefit of not retrying + the failed shrink a few times. This change ensures shrink no longer + sets max_retries and also makes all resize operations (shrink, clone, + split) leave the setting at default value rather than copy it from source. + +commit 3da91d5f7aa40b35f4ad50a961f785011fa7eed3 +Author: Hendrik Muhs +Date: Fri Oct 11 14:16:17 2019 +0200 + + [Transform] Rename internal indexes for transform plugin (#47788) (#47900) + + rename internal indexes of transform plugin + + - rename audit index and create an alias for accessing it, BWC: add an alias for old indexes to + keep them working, kibana UI will switch to use the read alias + - rename config index and provide BWC to read from old and new ones + +commit 5dd6bd6f494dea2145e4034128fbbcfd90aee3f7 +Author: Hendrik Muhs +Date: Fri Oct 11 12:27:54 2019 +0200 + + do not assert on state in mixed cluster due to endpoint differences (#47898) + + do not assert on state in mixed cluster due to endpoint differences between 7.3 and 7.4 + + regression #46452 + fixes #47693 + +commit 0c439fe4951fcc06944d7620c1ee66f577f976c1 +Author: Przemyslaw Gomulka +Date: Fri Oct 11 11:17:19 2019 +0200 + + [7.x] Allow partial parsing dates (#47872) backport(#46814) + + Enable partial parsing of date part. This is making the behaviour in java.time implementation the same as with joda. + 2018, 2018-01 and 2018-01-01 are all valid dates for date_optional_time or strict_date_optional_time + closes #45284 + closes #47473 + +commit fd1c4c198a9e2247a00106a0c71b57f641cf834d +Author: Hendrik Muhs +Date: Fri Oct 11 11:08:55 2019 +0200 + + [Transform] fixes tests which might fail due to auto-stop (#47867) + + Batch transforms automatically stop after all data has processed, therefore tests can not reliable test the state. This change rewrites tests to remove the unreliable tests or use continuous transforms instead as they do not auto-stop. + + fixes #47441 + +commit d210bfa8889890c6a95308e51181dbb81df22ae3 +Author: Przemysław Witek +Date: Fri Oct 11 10:19:55 2019 +0200 + + [7.x] Add MlClientDocumentationIT tests for classification. (#47569) (#47896) + +commit e60221d2bdf35d519e75bc050d9534404a7f30ce +Author: Alexander Reelsen +Date: Fri Oct 11 09:23:44 2019 +0200 + + Update jakarta mail dependency to 1.6.4 (#47810) + + This one contains a few small bugfixes, see https://eclipse-ee4j.github.io/mail/docs/CHANGES.txt + +commit ec91fa5df17711f8b4d81443b5d0d8f5088bcfcd +Author: Alpar Torok +Date: Fri Oct 11 04:29:19 2019 +0300 + + Make the run task honor tests.es properties (#47860) + + * Make the run task honor tests.es properties + + Closes #47797 + + With this PR we now again honor tests.es and tests.heap.size + + * remove debuging + +commit d315726c23a6a3fefe826a7352513a483931998d +Author: Matt Braymer-Hayes +Date: Thu Oct 10 14:20:24 2019 -0400 + + [DOCS] Fix gcloud command typo in discovery-gce.asciidoc. (#47312) + +commit 48823b1112050ebe084c037bbae76b2ef0f380ab +Author: Armin Braun +Date: Thu Oct 10 18:41:57 2019 +0200 + + Fix SLMSnapshotBlockingIntegTests (#47841) (#47863) + + One of the tests in this suit stops a master node, + plus we're doing other node starts in this suit. + => the internal test cluster should be TEST and not `SUITE` + scoped to avoid random failures like the one in #47834 + + Closes #47834 + +commit 2de3411c9cec763bf8c6b3f1f030cb60258c714f +Author: Zachary Tong +Date: Wed Jun 5 15:18:23 2019 -0400 + + Make sibling pipeline agg ctor's protected (#42808) + + SiblingPipelineAggregator is a public interfaces, + but the ctor was package-private. These should be protected so that + plugin authors can extend and implement their own sibling pipeline agg. + +commit 59b3294bc983488c7b17485430dbb39c8bafb6ee +Author: Marios Trivyzas +Date: Thu Oct 10 15:24:36 2019 +0200 + + SQL: Implement DATEADD function (#47747) + + Implement DATEADD/TIMESTAMPADD function as per the MS-SQL spec: + https://docs.microsoft.com/en-us/sql/t-sql/functions/dateadd-transact-sql?view=sql-server-2017 + which allows a user to add/subtract specified number of specified units + to/from a date/datetime field/expression. + + Closes: #47746 + (cherry picked from commit e624bc281bebb4bbe0b0c2e0a8cbc712e50097a8) + +commit b5afa95fd8827347740620a9aeca650872e47ef4 +Author: Igor Motov +Date: Thu Oct 10 18:04:58 2019 +0400 + + Fix Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart + + Tracked by #47612 + +commit 17433e79d8e865dd6b1f33fb37e866af982187f1 +Author: Igor Motov +Date: Thu Oct 10 17:49:58 2019 +0400 + + Mute RunDataFrameAnalyticsIT.testOutlierDetectionStopAndRestart + + Tracked by #47612 + +commit a8e027551ad66eb4ccee6e83c9b075b3e34c2738 +Author: James Rodewig +Date: Thu Oct 10 09:48:40 2019 -0400 + + [DOCS] Document `bytes` and `time` params in cat API docs (#47672) (#47854) + +commit dc6f0f9dc7baf3112e2401d7e89a3cb89f9ab95f +Author: Costin Leau +Date: Thu Oct 10 15:46:27 2019 +0300 + + SQL: Re-enable muted test + + Close #47080 + + (cherry picked from commit 63a0aa7b392f565ea01ac478fec1dd91a80202e5) + +commit 102016d5711e21e9b9d88ea59cbb3081e2a260c1 +Merge: aace42d38d9 f07de06cdde +Author: Martijn van Groningen +Date: Thu Oct 10 14:44:05 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit f07de06cdde72e8f11c26cb4e6c7ead2bbd6e54d +Author: Christoph Büscher +Date: Thu Oct 10 14:38:01 2019 +0200 + + Ensure random timestamps are within search boundary (#38753) (#47787) + + The random timestamps were landing too close to the current time, + so an unlucky rollup interval would round such that the doc wasn't + included in the search range (and thus not "rolled up") which + would then fail the test. + + The fix is to make sure the timestamp of all docs is sufficiently behind + 'now' that the possible rounding intervals will always include them. + + Backport of #38753 to 7.x where the test was still muted. + +commit c1f30e34ffaae704273d64e42de6cbfade2f0d2a +Author: Marios Trivyzas +Date: Thu Oct 10 13:30:23 2019 +0200 + + SQL: Refactor binary date time functions (#47786) + + Refactor DateTrunc and DatePart to use separate Pipe classes which + allows the removal of the BinaryDateOperation enum. + + (cherry picked from commit a6075e7718dff94a90dbc0795dd924dcb7641092) + +commit 6a4bf5de2c1e5fd3a5a3327c4f3bc9ac75c17041 +Author: Andrei Stefan +Date: Thu Oct 10 13:57:44 2019 +0300 + + SQL: make date/datetime and interval types compatible in conditional functions (#47595) + + (cherry picked from commit 6ff953e6396d7cc90640419aee5d036954e2eae3) + +commit 9eac8bf2a83fdea832f671a9bce5f58fee3c9592 +Author: István Zoltán Szabó +Date: Thu Oct 10 12:34:39 2019 +0200 + + [DOCS] Adds supported fields section to the PUT DFA API description (#47842) + +commit ecb20ebc6cb155be37ecdc68ca8a178531744758 +Author: David Turner +Date: Thu Oct 10 10:53:27 2019 +0200 + + More bootstrap docs tweaks (#47809) + + Clarifies not to set `cluster.initial_master_nodes` on nodes that are joining + an existing cluster. + + Co-Authored-By: James Rodewig + +commit dc39196ea407ce3d5e09ea77b51ba35c4f07a10a +Author: Jim Ferenczi +Date: Thu Oct 10 10:35:09 2019 +0200 + + Fix tag in the search request timeout option docs (#47776) + + and add missing parentheses `search_timeout` param + +commit bd6e2592a7bc921786b0c232438248cb2ce5482a +Author: Jim Ferenczi +Date: Wed Oct 9 14:40:05 2019 +0200 + + Remove the SearchContext from the highlighter context (#47733) + + Today built-in highlighter and plugins have access to the SearchContext through the + highlighter context. However most of the information exposed in the SearchContext are not needed and a QueryShardContext + would be enough to perform highlighting. This change replaces the SearchContext by the informations that are absolutely + required by highlighter: a QueryShardContext and the SearchContextHighlight. This change allows to reduce the exposure of the + complex SearchContext and remove the needs to clone it in the percolator sub phase. + + Relates #47198 + Relates #46523 + +commit 0e7869128a68a159641bdbb9165407f67f836d9f +Author: Hendrik Muhs +Date: Thu Oct 10 10:31:24 2019 +0200 + + [7.5][Transform] introduce new roles and deprecate old ones (#47780) (#47819) + + deprecate data_frame_transforms_{user,admin} roles and introduce transform_{user,admin} roles as replacement + +commit fbbe04b8010dfef63e2d5cf3fd772e0a3c4f6992 +Author: Alpar Torok +Date: Thu Oct 10 11:23:24 2019 +0300 + + Add a verifyVersions to the test FW (#47192) + + The test FW has a method to check that it's implementation of getting + index and wire compatible versions as well as reasoning about which + version is released or not produces the same rezults as the simillar + implementation in the build. + + This PR adds the `verifyVersions` task to the test FW so we have one + task to check everything related to versions. + +commit 3d334a262b5874a3e70516047a760b06b7b82411 +Author: Jim Ferenczi +Date: Thu Oct 10 09:59:19 2019 +0200 + + Ensure that we don't call listener twice when detecting a partial failure in _search (#47694) + + This change fixes a bug that can occur when a shard failure is detected while we build + the search response and accept partial failures in set to false. In this case we currently + call onFailure on the provided listener but also continue the search as if the failure didn't + occur. This can lead to a listener called twice, once with onFailure and once with onSuccess + which is forbidden by design. + +commit aace42d38d9245e48c868073d51fd026b9935dee +Author: Martijn van Groningen +Date: Thu Oct 10 08:46:30 2019 +0200 + + Add HLRC support for enrich stats API (#47306) + + This PR also includes HLRC docs for the enrich stats api. + + Relates to #32789 + +commit 19393fc5a7dd4e7c8082971d6a1b0760ce128b5f +Author: Martijn van Groningen +Date: Thu Oct 10 08:45:43 2019 +0200 + + match processor should handler values other than string properly (#47419) + + Currently if the document being ingested contains another field value + than a string then the processor fails with an error. + + This commit changes the match processor to handle number values + and array values correctly. + + If a json array is detected then the `terms` query is used instead + of the `term` query. + +commit 5825d2df8333afa44618f7ce27f8179ed9891859 +Author: Mark Vieira +Date: Wed Oct 9 15:42:49 2019 -0700 + + Resolve more Gradle task validation warnings (#47825) + + (cherry picked from commit 82e3d5ea31101f4c27e69162684c3aa4fb2193a4) + + # Conflicts: + # buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy + +commit 0360a18f61b57467bbbd083f2a1d426a6d3bcd24 +Author: Mark Vieira +Date: Wed Oct 9 15:37:54 2019 -0700 + + Mute test SLMSnapshotBlockingIntegTests.testRetentionWhileSnapshotInProgress + + Signed-off-by: Mark Vieira + (cherry picked from commit a8a7477c396554926f260d210364f009d85ae5f2) + +commit dc4224fbdf7dfb679041e067f57156e8af74292e +Author: dengweisysu +Date: Thu Oct 10 05:15:15 2019 +0800 + + Sync translog without lock before trim unreferenced readers (#47790) + + This commit is similar to the optimization made in #45765. With this + change, we fsync most of the data of the current generation without + holding writeLock when trimming unreferenced readers. + + Relates #45765 + +commit 302e09decfb4537c98eeef42cdde0de9fe7834af +Author: Armin Braun +Date: Wed Oct 9 23:29:50 2019 +0200 + + Simplify some Common ActionRunnable Uses (#47799) (#47828) + + Especially in the snapshot code there's a lot + of logic chaining `ActionRunnables` in tricky + ways now and the code is getting hard to follow. + This change introduces two convinience methods that + make it clear that a wrapped listener is invoked with + certainty in some trickier spots and shortens the code a bit. + +commit 14b979a7bcf814fca422414050b4f6e79da23b18 +Author: Ryan Ernst +Date: Wed Oct 9 12:00:59 2019 -0700 + + Make internal project flag overrideable by tests (#47757) + + In order to work with external elasticsearch plugins, some parts of + build-tools need to know when the current build is part of the + elasticsearch project or an external build. We identify these "internal" + builds by a marker in our buildSrc classpath. However, some build-tools + integ tests need to override this flag to test both the external and + internal behavior. + + This commit moves the storage of the flag indicating whether we are + running in an internal build to global build info. This will allow + testkit projects to override the flag. + +commit 9b3790d4f2582c5af516ed03261b117c04a5137d +Author: Gordon Brown +Date: Mon Sep 30 13:08:10 2019 -0600 + + Mute "Test All Indexes Lifecycle Explain" (#47317) + +commit 076d3073b56c55685aa476692f0e9ad8c4b3aa5b +Author: Jack Conradson +Date: Wed Oct 9 10:24:19 2019 -0700 + + Move binding member field generation to Painless semantic pass (#47739) + + This adds an SField node that operates similarly to SFunction as a top level + node meant only for use in an SClass node. Member fields are generated + for both class bindings and instance bindings using the new SField node + during the semantic pass, and information is no longer passed through + Globals for this during the write pass. + +commit 8f86469d3fbf540dc1e13a6d572f8d19806d9231 +Author: Tanguy Leroux +Date: Wed Oct 9 19:16:23 2019 +0200 + + Do not auto-follow closed indices (#47721) (#47800) + + Backport of (#47721) for 7.x. + + Similarly to #47582, Auto-follow patterns creates following + indices as long as the remote index matches the pattern and + the remote primary shards are all started. But since 7.2 closed + indices are also replicated, and it does not play well with CCR + auto-follow patterns as they create following indices for closed + leader indices too. + + This commit changes the getLeaderIndicesToFollow() so that closed + indices are excluded from auto-follow patterns. + +commit 813f44cec8f9d190fec65817deb70ec7b5508566 +Author: Ryan Ernst +Date: Wed Oct 9 09:58:59 2019 -0700 + + Move test seed setup to global build info (#47763) + + The global build info plugin prints high level information about the + build, including the test seed. However, BuildPlugin sets up the test + seed, which creates an odd implicit dependency on it. This commit moves + the intialization of the testSeed property into the global build info. + +commit 12e4e7ef545fa1a567fcaf423ab80d9a0936753a +Author: Igor Motov +Date: Wed Oct 9 14:35:54 2019 +0400 + + Geo: implement proper handling of out of bounds geo points (#47734) + + This is the first iteration in improving of handling of out of + bounds geopoints with a latitude outside of the -90 - +90 range + and a longitude outside of the -180 - +180 range. + + Relates to #43916 + +commit f8b8afdc70211973f04cd411cfe32a1a88bcd783 +Author: Igor Motov +Date: Wed Oct 9 14:35:10 2019 +0400 + + Geo: Fixes indexing of linestrings that go around the globe (#47471) + + LINESTRING (0 0, 720 20) is now decomposed into 3 strings: + multilinestring ( + (0.0 0.0, 180.0 5.0), + (-180.0 5.0, 180 15), + (-180.0 15.0, 0 20) + ) + + It also fixes issues with linestrings that intersect antimeridian + more than 5 times. + + Fixes #43837 + Fixes #43826 + +commit d18ff24dbed6ec1bc74d452b3ae4c41fc0d263e9 +Author: Tim Brooks +Date: Tue Aug 13 19:44:39 2019 -0400 + + Fix BulkByScrollResponseTests exception assertions (#45519) + + Currently in the x content serialization tests we compare the exception + messages that are serialized. These exceptions messages are not + equivalent because the exception often changes when serialized to x + content. This commit removes this assertion. + +commit 6f4b7e9a7fa38917d410ceb2dfbcdaaee7acd2a0 +Author: István Zoltán Szabó +Date: Wed Oct 9 18:13:33 2019 +0200 + + [DOCS] Extends the analyzed_fields description in the PUT DFA API docs (#47791) + +commit 02622c1ef95e4e40f44fb8a4c622e15ef9c15534 +Author: Tim Brooks +Date: Tue Aug 13 15:55:01 2019 -0400 + + Fix issues with serializing BulkByScrollResponse (#45357) + + Currently there are two issues with serializing BulkByScrollResponse. + First, when deserializing from XContent, indexing exceptions and search + exceptions are switched. Additionally, search exceptions do no retain + the appropriate RestStatus code, so you must evaluate the status code + from the exception. However, the exception class is not always correctly + retained when serialized. + + This commit adds tests in the failure case. Additionally, fixes the + swapping of failure types and adds the rest status code to the search + failure. + +commit d96977202db6dbaf0915bf280283747afec18ddb +Author: Jim Ferenczi +Date: Wed Oct 9 10:51:38 2019 +0200 + + Disable SLMSnapshotBlockingIntegTests#testSnapshotInProgress (#47775) + + This test fails constantly in master and prs. + + Relates #47689 + +commit 43dc72f1a56ccbd502c818ed24a01c6dcfc6123d +Author: Jake Landis +Date: Wed Oct 9 10:47:21 2019 -0500 + + Fix cluster alert for watcher/monitoring IndexOutOfBoundsExcep… (#47756) + + If a cluster sending monitoring data is unhealthy and triggers an + alert, then stops sending data the following exception [1] can occur. + + This exception stops the current Watch and the behavior is actually + correct in part due to the exception. Simply fixing the exception + introduces some incorrect behavior. Now that the Watch does not + error in the this case, it will result in an incorrectly "resolved" + alert. The fix here is two parts a) fix the exception b) fix the + following incorrect behavior. + + a) fixing the exception is as easy as checking the size of the + array before accessing it. + + b) fixing the following incorrect behavior is a bit more intrusive + + - Note - the UI depends on the success/met state for each condition + to determine an "OK" or "FIRING" + + In this scenario, where an unhealthy cluster triggers an alert and + then goes silent, it should keep "FIRING" until it hears back that + the cluster is green. To keep the Watch "FIRING" either the index + action or the email action needs to fire. Since the Watch is neither + a "new" alert or a "resolved" alert, we do not want to keep sending + an email (that would be non-passive too). Without completely changing + the logic of how an alert is resolved allowing the index action to + take place would result in the alert being resolved. Since we can + not keep "FIRING" either the email or index action (since we don't + want to resolve the alert nor re-write the logic for alert resolution), + we will introduce a 3rd action. A logging action that WILL fire when + the cluster is unhealthy. Specifically will fire when there is an + unresolved alert and it can not find the cluster state. + This logging action is logged at debug, so it should be noticed much. + This logging action serves as an 'anchor' for the UI to keep the state + in an a "FIRING" status until the alert is resolved. + + This presents a possible scenario where a cluster starts firing, + then goes completely silent forever, the Watch will be "FIRING" + forever. This is an edge case that already exists in some scenarios + and requires manual intervention to remove that Watch. + + This changes changes to use a template-like method to populate the + version_created for the default monitoring watches. The version is + set to 7.5 since that is where this is first introduced. + + Fixes #43184 + +commit f8ebb75fcfd8b313cd69affe2166606abf87e7b2 +Author: Martijn van Groningen +Date: Wed Oct 9 17:31:10 2019 +0200 + + Reuse OperationRouting#searchShards(...) to select local enrich shard (#47359) + + The currently logic shard selecting logic selects a random shard copy + instead of selecting the local shard copy and if local copy is not + available then selecting a random shard copy. The latter is desired + behaviour for enrich. + + By reusing `OperationRouting#searchShards(...)` we get the desired + behaviour and reuse the same logic that the search api is using. + +commit 65f8294378e808b87d3ee074218d6450a061b05b +Author: James Rodewig +Date: Wed Oct 9 08:39:11 2019 -0400 + + [DOCS] Add docs for `geo_match` enrich policy type (#47745) + +commit 2abd9d53b6af5b09d5c74cad0f32e3fbbfd24512 +Author: Florian Kelbert +Date: Wed Oct 9 14:27:22 2019 +0200 + + [DOCS] Correct split API request format (#47774) + +commit 57671f607713b8fd9bf5c29e0c409eedc3c76415 +Author: Alpar Torok +Date: Wed Oct 9 14:26:26 2019 +0300 + + Enable 7.x to run with --parallel + +commit edc3e9f0abfdd419b2bd48f07a69a497e0f43bd6 +Author: Armin Braun +Date: Wed Oct 9 13:25:16 2019 +0200 + + Fix --debug-jvm Gradle Arg (#47773) (#47783) + + This fixes the `--debug-jvm` arg to work again for + the `run` task. + Seems a recent refactoring of `RunTask` introduced + this obvious type. + +commit 1139cce9a3cc38ef20dcaf809e7264a11bd28a30 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Oct 9 21:22:36 2019 +1100 + + [DOCS] Add docs for `create_doc` index privilege (#47584) (#47778) + + This commit adds documentation for new index privilege + create_doc which only allows indexing of new documents + but no updates to existing documents via Index or Bulk APIs. + + Relates: #45806 + +commit 75a7daae73cca0cb7c3d0afe576ed071c1699690 +Author: Andrei Stefan +Date: Wed Oct 9 11:22:41 2019 +0300 + + SQL: use calendar interval of 1y instead of fixed interval for grouping by YEAR and HISTOGRAMs (#47558) + + (cherry picked from commit 55f5463eee4ecea3537df4b34645f1d87472a802) + +commit be0e17770ca1c531d8bd303034aa0ca52448b371 +Author: Martijn van Groningen +Date: Wed Oct 9 09:16:23 2019 +0200 + + required change after merging in 7 dot x branch + +commit da1e2ea4613323dc99142c716b8d21938f934973 +Merge: b9fb3546183 54c2aec38ac +Author: Martijn van Groningen +Date: Wed Oct 9 09:06:13 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 54c2aec38ac1b210bd7833f32d373e6ccd6e9593 +Author: Ryan Ernst +Date: Tue Oct 8 16:59:26 2019 -0700 + + Filter out special gradle threads from leak control (#47713) + + This commit adds a thread filter for gradle unit tests which omits + threads gradle may create that we have no control over shutting down. + The current example of this is for project.exec which gradle pools. + + closes #47417 + +commit fb7abe9fa4162b07bfa70822eff64e4385edf282 +Author: Lee Hinman +Date: Tue Oct 8 17:15:49 2019 -0600 + + Separate SLM stop/start/status API from ILM (#47710) + + * Separate SLM stop/start/status API from ILM + + This separates a start/stop/status API for SLM from being tied to ILM's + operation mode. These APIs look like: + + ``` + POST /_slm/stop + POST /_slm/start + GET /_slm/status + ``` + + This allows administrators to have fine-grained control over preventing + periodic snapshots and deletions while performing cluster maintenance. + + Relates to #43663 + + * Allow going from RUNNING to STOPPED + + * Align with the OperationMode rules + + * Fix slmStopping method + + * Make OperationModeUpdateTask constructor private + + * Wipe snapshots better in test + +commit a492864a9d093857ff9f8d26d1b0f1c3ffe50bc5 +Author: Gordon Brown +Date: Tue Oct 8 17:07:08 2019 -0600 + + Manage retention of failed snapshots in SLM (#47617) + + Failed snapshots will eventually build up unless they are deleted. While + failures may not take up much space, they add noise to the list of + snapshots and it's desirable to remove them when they are no longer + useful. + + With this change, failed snapshots are deleted using the following + strategy: `FAILED` snapshots will be kept until the configured + `expire_after` period has passed, if present, and then be deleted. If + there is no configured `expire_after` in the retention policy, then they + will be deleted if there is at least one more recent successful snapshot + from this policy (as they may otherwise be useful for troubleshooting + purposes). Failed snapshots are not counted towards either `min_count` + or `max_count`. + +commit a6b0ae7f6903428250dc5656578d8d5f2f7b9d24 +Author: Jake Landis +Date: Tue Oct 8 17:23:38 2019 -0500 + + Fix bug in ingest node documentation (#45589) (#47750) + + The "Conditionals with the Pipeline Processor" incorrectly documents + how to create a pipeline of pipelines with a failure condition. The + example as-is will always execute the fail processor. The change here + updates the documentation to correct guard the fail processor with an + if condition. + +commit 96b36b5a8ce03ad40b4be2e1780cfdb1b1d17705 +Author: Armin Braun +Date: Tue Oct 8 21:04:51 2019 +0200 + + Make loadShardSnapshot Exceptions Consistent (#47728) (#47735) + + Similar to #47507. We are throwing `SnapshotException` when + you (and SLM tests) would expect a `SnapshotMissingException` + for concurrent snapshot status and snapshot delete operations + with a very low probability. + Fixed the exception type and added a test for this scenario. + +commit b9fb3546183e0513c61ed3a09b2bf9e420c4bd76 +Author: James Baiera +Date: Fri Oct 4 15:45:16 2019 -0400 + + Add retry to force merge operation in EnrichPolicyRunner (#47178) + + Adds a check when running an Enrich policy to make sure that an Enrich index + is force merged down to one segment, and if it was not fully merged, attempts + the merge again, up to a configurable number of times. + +commit f4c884450f3efb85fd0f4f00519f3d0aead1705f +Author: Ioannis Kakavas +Date: Tue Oct 8 18:14:41 2019 +0300 + + Remove unmerged changes from release notes (#47732) + +commit 8b7100eb1fbd6ed8fa29a4fe566420ced5045668 +Author: Martijn van Groningen +Date: Tue Oct 8 17:10:16 2019 +0200 + + Don't remove indices to avoid monitoring from intermittently failing + to index monitoring docs. + +commit 5cef4752f743b8a52da89c39faed93f5d9694ea9 +Author: Armin Braun +Date: Tue Oct 8 17:01:54 2019 +0200 + + Fix Ex. Handling in SnapshotsService#snapshots (#47507) (#47727) + + We're needlessly wrapping a `SnapshotMissingException` which + itself is a `SnapshotException` when trying to load a missing + snapshot. This leads to failure #47442 which expects a + `SnapshotMissingException` in this case. + + Closes #47442 + +commit ea4069ca63fc0ffeac82ab2430052f78dd39e3db +Author: Lee Hinman +Date: Tue Oct 8 08:32:15 2019 -0600 + + Add Snapshot Lifecycle Retention documentation (#47545) + + * Add Snapshot Lifecycle Retention documentation + + This commits adds API and general purpose documentation for SLM + retention. + + Relates to #43663 + + * Fix docs tests + + * Update default now that #47604 has been merged + + * Update docs/reference/ilm/apis/slm-api.asciidoc + + Co-Authored-By: Gordon Brown + + * Update docs/reference/ilm/apis/slm-api.asciidoc + + Co-Authored-By: Gordon Brown + + * Update docs with feedback + +commit b578059c9005902a0009db3b86c84ef53018618f +Author: Jake Landis +Date: Tue Oct 8 09:45:27 2019 -0500 + + Re-enable Watcher rest test (#47699) (#47705) + + This test is believed to be fixed by #43939 + + closes #43988 + +commit 11093197f149262edac4243e7825d60c49e3e283 +Author: David Turner +Date: Tue Oct 8 15:41:34 2019 +0200 + + Fix deprecation docs formatting (#47725) + + Relates #47443 + +commit c1b0bfd74aff0691b3dc5ed74049fd1f48d38a7b +Author: Dimitris Athanasiou +Date: Tue Oct 8 16:02:47 2019 +0300 + + [7.x][ML] Unwrap exception causes before calling instanceof (#47676) (#47724) + + When exceptions could be returned from another node, the exception + might be wrapped in a `RemoteTransportException`. In places where + we handled specific exceptions using `instanceof` we ought to unwrap + the cause first. + + This commit attempts to fix this issue after searching code in the ML + plugin. + + Backport of #47676 + +commit 9b4eec9887a1c2cfd91bbda838d70b02dde6f3a4 +Author: James Rodewig +Date: Tue Oct 8 08:34:39 2019 -0400 + + [DOCS] Fix errors in rollover index API docs (#47702) + +commit 36d018c90907d1f7da24bd8449248e81b17c2530 +Author: Alpar Torok +Date: Tue Oct 8 14:39:58 2019 +0300 + + Convert RunTask to use testclusers, remove ClusterFormationTasks (#47572) + + * Convert RunTask to use testclusers, remove ClusterFormationTasks + + This PR adds a new RunTask and a way for it to start a + testclusters cluster out of band and block on it to replace + the old RunTask that used ClusterFormationTasks. + + With this we can now remove ClusterFormationTasks. + +commit d33dbf82d47ee4962cdf6c3616dbf9f57d5e3282 +Author: Benjamin Trent +Date: Tue Oct 8 07:11:05 2019 -0400 + + [7.x] [ML][Inference] adjusting definition object schema and validation (#47447) (#47673) + + * [ML][Inference] adjusting definition object schema and validation (#47447) + + * [ML][Inference] adjusting definition object schema and validation + + * finalizing schema and fixing inference npe + + * addressing PR comments + + * fixing for backport + +commit ce91ba7c2550f0d5a4a9748076b78d2d37887321 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Oct 8 12:08:50 2019 +0200 + + Dangling indices strip aliases (#47581) + + Importing dangling indices with aliases risks breaking functionalities + using those aliases. For instance, writing to an alias may break if + there is no is_write_index indication on the existing alias and the + dangling index import adds a second index to the alias. Or an + application could have an assumption about the alias only ever pointing + to one index and suddenly seeing the alias also linked to an old index + could break it. With this change we strip aliases of the index meta data + found before importing a dangling index. + +commit bb5f750ab437fc2e67fb35e1f4c2dfb08a850ca7 +Author: David Turner +Date: Tue Oct 8 09:15:13 2019 +0200 + + Deprecate include_relocations setting (#47443) + + Setting `cluster.routing.allocation.disk.include_relocations` to `false` is a + bad idea since it will lead to the kinds of overshoot that were otherwise fixed + in #46079. This commit deprecates this setting so it can be removed in the next + major release. + +commit 5e0e54f4554605ac9e6a2d0b9355b60693e932dd +Author: Hendrik Muhs +Date: Tue Oct 8 08:59:01 2019 +0200 + + [Transform] move root endpoint to _transform with BWC layer (#47127) (#47682) + + move the main endpoint to /_transform/ from /_data_frame/transforms/ with providing backwards compatibility and deprecation warnings + +commit 91988c7c2683ddcb8b2cd3eceb3d39330b32f6ae +Author: Lee Hinman +Date: Mon Oct 7 19:53:46 2019 -0600 + + Throw error retrieving non-existent SLM policy (#47679) + + Previously when retrieving an SLM policy it would always return a 200 + with `{}` in the body, even if the policy did not exist. This changes + that behavior to throw an error (similar to our other APIs) if a + policy doesn't exist. + + This also adds a basic CRUD yml test for the behavior. + + Resolves #47664 + +commit 906be4520915b2126cd76c4ef6f30ebea6c4cff4 +Author: Lee Hinman +Date: Mon Oct 7 19:31:17 2019 -0600 + + Add a test for SLM retention with security enabled (#47608) + + This enhances the existing SLM test using users/roles/etc to also test + that SLM retention works when security is enabled. + + Relates to #43663 + +commit 8c6d1e0a08931bb3c9037da08f6107ac3cd55956 +Author: Ryan Ernst +Date: Mon Oct 7 17:08:37 2019 -0700 + + Switch stored script example to script_score query (#47691) + + The example use of a scoring script was incorrectly using a filter + script query, which has no scoring, and thus no _score variable + avialable. This commit converts the example doc to using the newer + script_score query. + +commit 39ef7950856f54fbf853f2f2d67dd32371a224d1 +Author: Lisa Cawley +Date: Mon Oct 7 15:23:19 2019 -0700 + + [DOCS] Cleans up links to security content (#47610) (#47703) + +commit a17f394e27e95c62b66440f80eab2d27433468d5 +Author: Tal Levy +Date: Mon Oct 7 15:03:46 2019 -0700 + + Geo-Match Enrich Processor (#47243) (#47701) + + this commit introduces a geo-match enrich processor that looks up a specific + `geo_point` field in the enrich-index for all entries that have a geo_shape match field + that meets some specific relation criteria with the input field. + + For example, the enrich index may contain documents with zipcodes and their respective + geo_shape. Ingesting documents with a geo_point field can be enriched with which zipcode + they associate according to which shape they are contained within. + + this commit also refactors some of the MatchProcessor by moving a lot of the shared code to + AbstractEnrichProcessor. + + Closes #42639. + +commit 74876811c2b17a89bf2bbf70bdd250040b1f7056 +Author: Jake Landis +Date: Mon Oct 7 15:45:45 2019 -0500 + + Watcher - catch uncaught exception. (#47680) (#47695) + + If a thread pool rejection exception happens, an alternative code + path is chosen to write history and delete the trigger. If an exception + happens during deletion of the trigger an exception may be thrown and not + caught. + + This commit catches the exception and provides a meaning error message. + + fixes #47008 + +commit a49a1b6994c96e328b28db5f6b1825a3087fed4b +Author: Jake Landis +Date: Mon Oct 7 12:07:10 2019 -0500 + + Watcher remove assertion that is susceptible to a race conditi… (#47667) + + When deactivating a watch, there is a chance that it is fully deactivated + and reporting as not running but the history is not fully written yet. + There is not a tight coupling between the associated watcher history + index and the deactivation. This test assumes that once a watch is + deactivated that all history is fully written in a very short time period. + If the Watch is deactivated, but the history is slow to write it can result + in a failing test. + + This change removes an assertion that assumes that the deactivation of a watch + ensured the all of the watch history was written. There is still a minor race + condition with respect to the remaining history assertions. However, if the + history is slow to be written, it will allow the test to still passing. + + fixes #47503 + +commit 7667ea5f6fc13200ddffc2954f49c41d141ca9f4 +Author: Dimitris Athanasiou +Date: Mon Oct 7 18:21:33 2019 +0300 + + [7.x][ML] Additional outlier detection parameters (#47600) (#47669) + + Adds the following parameters to `outlier_detection`: + + - `compute_feature_influence` (boolean): whether to compute or not + feature influence scores + - `outlier_fraction` (double): the proportion of the data set assumed + to be outlying prior to running outlier detection + - `standardization_enabled` (boolean): whether to apply standardization + to the feature values + + Backport of #47600 + +commit 833ed30f0dc04de5cf02a156498e4bd08df24629 +Author: Jack Conradson +Date: Mon Oct 7 07:47:56 2019 -0700 + + Modify Painless AST to add synthetic functions during semantic pass (#47611) + + This has ELambda and ENewArrayFunctionRef add their generated synthetic + methods to the SClass node during the semantic pass and removes this + data from the write pass. This is the first step to remove "Globals" (mutable + state) from the write pass. + +commit 176ca13c57fb72d05cb7c4864999a907b1db4548 +Author: James Rodewig +Date: Mon Oct 7 09:37:23 2019 -0400 + + [DOCS] Correct deprecation note in mapping docs (#47656) + +commit fc3cc30008281d0b473bda200bbb2bd1e027b170 +Author: James Rodewig +Date: Mon Oct 7 09:36:30 2019 -0400 + + [DOCS] Reformat clear cache API docs (#46512) (#47662) + +commit c03cdb4b154f8bb190e89d37cbb2a797aaf3c783 +Author: James Rodewig +Date: Mon Oct 7 09:25:03 2019 -0400 + + [DOCS] Correct callouts in search template docs (#47655) + +commit e698e68f06f4cc7947f4d959f0254c114878093d +Author: Marios Trivyzas +Date: Mon Oct 7 14:37:20 2019 +0200 + + SQL: Allow whitespaces in escape patterns (#47577) + + Previously, we supported only the format `{fn ()}` + but other DBs like MSSQL, DB2, MariaDB/MySQL alos allow whitespaces + between `{` and `fn`. Furhermore, also some applications - like PowerBI - + generate escape sequences with spaces: `select { fn name(params) } etc.` + + Add support for white spaces between `{` and the escape pattern definition + like `fn`, `ts`, `d`, `guid` etc. + + Closes: #47401 + + (cherry picked from commit 08a22d0b393f4a76c52dabc5e7b9cafcc19c30ca) + +commit b6d1d2e6ec9ec06d26af5d56f3c353482f792fcc +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Oct 7 23:58:44 2019 +1100 + + Add 'create_doc' index privilege (#45806) (#47645) + + Use case: + User with `create_doc` index privilege will be allowed to only index new documents + either via Index API or Bulk API. + + There are two cases that we need to think: + - **User indexing a new document without specifying an Id.** + For this ES auto generates an Id and now ES version 7.5.0 onwards defaults to `op_type` `create` we just need to authorize on the `op_type`. + - **User indexing a new document with an Id.** + This is problematic as we do not know whether a document with Id exists or not. + If the `op_type` is `create` then we can assume the user is trying to add a document, if it exists it is going to throw an error from the index engine. + + Given these both cases, we can safely authorize based on the `op_type` value. If the value is `create` then the user with `create_doc` privilege is authorized to index new documents. + + In the `AuthorizationService` when authorizing a bulk request, we check the implied action. + This code changes that to append the `:op_type/index` or `:op_type/create` + to indicate the implied index action. + +commit 7c862fe71f3d44521ac5a8c844cfa12c18f43ffb +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Oct 7 23:58:21 2019 +1100 + + Add support to retrieve all API keys if user has privilege (#47274) (#47641) + + This commit adds support to retrieve all API keys if the authenticated + user is authorized to do so. + This removes the restriction of specifying one of the + parameters (like id, name, username and/or realm name) + when the `owner` is set to `false`. + + Closes #46887 + +commit f93bb9dac50df89596d9e43b0e05786ecf2c7ed9 +Author: James Rodewig +Date: Mon Oct 7 08:49:42 2019 -0400 + + [DOCS] Reformat type exists API docs (#47601) + +commit 63bb4c91eb726e7d1b1c3b33d314fb3b6659d028 +Author: Przemyslaw Gomulka +Date: Mon Oct 7 14:41:22 2019 +0200 + + Update deprecation logging doc with logger configuration (#47649) Backport#47508 + + Explicitly adds a configuration snippet to change logging level + +commit b669b8f0464bd90cbc14a033889742da391dbee0 +Author: Armin Braun +Date: Mon Oct 7 14:18:41 2019 +0200 + + Simplify Snapshot Delete Further (#47626) (#47644) + + This change removes the special path for deleting + the index metadata blobs and moves deleting them to + the bulk delete of unreferenced blobs at the end of + the snapshot delete process. + This saves N RPC calls for a snapshot containing N + indices and simplifies the code. + Also, this change moves the unreferenced data cleanup up + the stack to make it more obvious that any exceptions during + this pahse will be ignored and not fail the delete request. + Lastly, this change removes the needless chaining of + first deleting unreferenced data from the snapshot delete + and then running the stale data cleanup (that would also run + from the cleanup endpoint) and simply fires off the cleanup + right after updating the repository data (index-N) in parallel + to the other delete operations to speed up the delete some more. + +commit b5ac0204d2df9c8c0789f8abea8bc0ce88e45441 +Author: Tanguy Leroux +Date: Mon Oct 7 13:59:04 2019 +0200 + + Fail earlier Put Follow requests for closed leader indices (#47637) + + Backport of (#47582) + + Today when following a new leader index, we fetch the remote cluster state, + check the remote cluster license, check the user privileges, retrieve the + index shard stats before initiating a CCR restore session. + + But if the leader index to follow is closed, we're executing a bunch of + operations that would inevitability fail at some point (on retrieving the + index shard stats, because this type of request forbid closed indices + when resolving indices). We could fail a Put Follow request at the first + step by checking the leader index state directly from the remote cluster + state. + + This also helps the Resume Follow API to fail a bit earlier. + +commit bc85b22c1f3758bce2753607b826674994864023 +Author: Alpar Torok +Date: Mon Oct 7 11:43:57 2019 +0300 + + Complete testclusters backport (#47623) + + * Use versions specific distribution folders so we don't need to clean up (#46539) + + * Retry deleting distro dir on windows + + When retarting the cluster we clean up old distribution files that might + still be in use by the OS. + Windows closes resources of ded processes async, so we do a couple of + retries to get arround it. + + Closes #46014 + + * Avoid having to delete the distro folder. + + * Remove the use of ClusterFormationTasks form RestTestTask (#47022) + + This PR removes a use-case of the ClusterFormationTasks and converts a + project that flew under the radar so far. + There's probably more clean-up possible here, but for now the goal is + to be able to remove that code after `RunTask` is also updated. + + * Migrate some 7.x only projects + +commit 1359ef73a30e65eff734193bf11a738c93cf29df +Author: Armin Braun +Date: Mon Oct 7 10:38:19 2019 +0200 + + Add IT for Snapshot Issue in 47552 (#47627) (#47634) + + * Add IT for Snapshot Issue in 47552 (#47627) + + Adding a specific integration test that reproduces the problem + fixed in #47552. The issue fixed only reproduces in the snapshot + resiliency otherwise which are not available in 6.8 where the + fix is being backported to as well. + +commit 6bd033931be1be8b43f5a0ef01c6106d80274384 +Author: Armin Braun +Date: Mon Oct 7 10:37:56 2019 +0200 + + Add Consistency Assertion to SnapshotsInProgress (#47598) (#47633) + + Assert given input shards and indices are consistent. + Also, fixed the equality check for SnapshotsInProgress. + Before this change the tests never had more than a single waiting + shard per index so they never failed as a result of the + waiting shards list not being ordered. + + Follow up to #47552 + +commit 736fceb18b59e2afbffc9f5f7258e1c1b1aa4ca7 +Author: Luca Cavanna +Date: Mon Oct 7 10:10:04 2019 +0200 + + Fold InitialSearchPhase into AbstractSearchAsyncAction (#47182) + + Historically, we have two base classes for search actions that generally need to fan out to multiple shards and then move on to the following phase: InitialSearchPhase and AbstractSearchAsyncAction that extends it. Practically, every search action extends the latter, and there are no direct subclasses of InitialSearchPhase in our codebase. + + This commit folds InitialSearchPhase into AbstractSearchAsyncAction in the attempt of simplifying things and making the search code running on the coordinating node easier to reason about. + +commit f2f2304c75d0add960f445ed514375fcd5a2a629 +Merge: a66c0dcd957 4506b37ed5d +Author: Martijn van Groningen +Date: Mon Oct 7 10:07:56 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 4506b37ed5dd72cabbc648c9b1daf7b00020009c +Author: Andrei Dan +Date: Mon Oct 7 07:47:47 2019 +0100 + + ILM: Skip rolling indexes that are already rolled (#47324) (#47592) + + An index with an ILM policy that has a rollover action in one of the + phases was rolled over when the ILM conditions dictated regardless if + it was already rolled over (eg. manually after modifying an index + template in order to force the creation of a new index that uses the new + mappings). + This changes this behaviour and has ILM check if the index it's about to + roll has not been rolled over in the meantime. + + (cherry picked from commit 37d6106feeb9f9369519117c88a9e7e30f3ac797) + Signed-off-by: Andrei Dan + +commit 36cabbae800f65917abc44a93e30a4410158a6f5 +Author: Ioannis Kakavas +Date: Mon Oct 7 09:19:32 2019 +0300 + + NameID mapping and Single Logout (#47288) (#47561) + + Clarify in the documentation that for SAML Single Logout to be + functional, the Identity Provider needs to release a NameID. + +commit 41c04ef39c14ee1efe15cd2a0d920e1d6c3f64a8 +Author: debadair +Date: Sun Oct 6 22:25:21 2019 -0700 + + [DOCS] Backporting API ref reformatting for document APIs (#47631) + + * [DOCS] Reformats bulk API. (#47479) + + * Reformats bulk API. + + * Update docs/reference/docs/bulk.asciidoc + + Co-Authored-By: James Rodewig + + * Reformats mget API (#47477) + + * Reformats mget API + + * Update docs/reference/docs/get.asciidoc + + Co-Authored-By: James Rodewig + + * Incorporated feedback. + + * Reformats reindex API (#47483) + + * Reformats reindex API + + * Incorporated review feedback. + + * Reformats term vectors APIs (#47484) + + * Reformat termvectors APIs + + * Reformats mtermvectors + + * Apply suggestions from code review + + Co-Authored-By: James Rodewig + + * Incorporated review feedback. + +commit ffacfc642c1d56ddee0bf3d83bf5013d89ec520b +Author: Dimitris Athanasiou +Date: Sat Oct 5 23:58:32 2019 +0300 + + [7.x][ML] Mute RegressionIT.testStopAndRestart (#47624) (#47625) + + Relates #47612 + +commit f765ded65092533d51291f0acf44abc9c5239f29 +Author: debadair +Date: Sat Oct 5 09:47:32 2019 -0700 + + [DOCS] Comment out tag in Task Managment API Docs so it isn't rendered. (#47618) + + The tag for the shared content is being rendered in the output. + +commit 22679c793211479d07c61924684a5750fde9e50d +Author: Armin Braun +Date: Sat Oct 5 15:01:06 2019 +0200 + + Fix Snapshot Corruption in Edge Case (#47552) (#47620) + + This fixes missing to marking shard snapshots as failures when + multiple data-nodes are lost during the snapshot process or + shard snapshot failures have occured before a node left the cluster. + + The problem was that we were simply not adding any shard entries for completed + shards on node-left events. This has no effect for a successful shard, but + for a failed shard would lead to that shard not being marked as failed during + snapshot finalization. Fixed by corectly keeping track of all previous completed + shard states as well in this case. + Also, added an assertion that without this fix would trip on almost every run of the + resiliency tests and adjusted the serialization of SnapshotsInProgress.Entry so + we have a proper assertion message. + + Closes #47550 + +commit f2d2ca21e2e3c5a6faa0a192b7e90b82fb4589ca +Author: Armin Braun +Date: Sat Oct 5 09:45:10 2019 +0200 + + Cleaner Handling of Store Refcount in BlobStoreRepository (#47560) (#47594) + + If a shard gets closed we properly abort its snapshot + before closing it. We should in thise case make sure to + not throw a confusing exception about trying to increment + the reference on an already closed shard in the async tasks + if the snapshot is already aborted. + Also, added an assertion to make sure that aborts are in + fact the only situation in which we run into a concurrently + closed store. + +commit e47bdf760e2fd138c33d4b178bfd933e89dafc94 +Author: Gordon Brown +Date: Fri Oct 4 17:40:05 2019 -0600 + + Fix Rollover error when alias has closed indices (#47148) (#47539) + + Rollover previously requested index stats for all indices in the + provided alias, which causes an exception when there is a closed index + with that alias. + + This commit adjusts the IndicesOptions used on the index stats + request so that closed indices are ignored, rather than throwing + an exception. + +commit 43f588a29e2921691c0f22e1dcda3e5f1638a7f0 +Author: Jason Tedor +Date: Fri Oct 4 19:29:20 2019 -0400 + + Fix compilation in JVM options parser code + + Compilation was accidentally broken here when a backport used code from + JDK 9, which is not supported in 7.x. This commit addresses this by + using JDK 8 compatiable APIs. + +commit 8a7e5b08477730d9a8e15f56e8e82530e8c8515c +Author: Jason Tedor +Date: Fri Oct 4 19:11:54 2019 -0400 + + Move ES_TMPDIR substitution into jvm options parser (#47189) + + This commit moves the ES_TMPDIR substitution that we do for JVM options + into the JVM options parser itself. This solves a problem where the fact + that the we do not make the substitution before ergonomics parsing can + lead to the JVM that we start for computing the ergonomic values failing + to start. Additionally, moving this substitution here enables us to + simplify the shell scripts since we do not need to implement this there, + and twice for Bash and Windows. + +commit d966e5a9b9d24ac4c7d0a7f8b9eb2e459ab2f567 +Author: Mark Vieira +Date: Fri Oct 4 13:12:42 2019 -0700 + + Eliminate Gradle task input warnings (#47538) + + (cherry picked from commit e86d40ff4576fb20c64fe88f01f13e201f3b948f) + +commit 35ca3d68d7a0640d72f47c05dc6a054f8b21bc1a +Author: Jason Tedor +Date: Fri Oct 4 17:32:49 2019 -0400 + + Validating monitoring hosts setting while parsing (#47571) + + This commit lifts the validation of the monitoring hosts setting into + the setting itself, rather than when the setting is used. This prevents + a scenario where an invalid value for the setting is accepted, but then + later fails while applying a cluster state with the invalid setting. + +commit e404f7ea80597bb203bdbd5538c0c2b32aae76d8 +Author: Mark Tozzi +Date: Fri Oct 4 17:21:17 2019 -0400 + + DocValueFormat implementation for date range fields (#47472) (#47605) + +commit 79376b72199350f0efaab7f4c46314e25d268e6b +Author: Lee Hinman +Date: Fri Oct 4 14:57:23 2019 -0600 + + Set default SLM retention invocation time (#47604) + + This adds a default for the `slm.retention_schedule` setting, setting it + to `0 30 1 * * ?` which is 1:30am every day. + + Having retention unset meant that it would never be invoked and clean up + snapshots. We determined it would be better to have a default than never + to be run. When coming to a decision, we weighed the option of an + absolute time (such as 1:30am) versus a periodic invocation (like every + 12 hours). In the end we decided on the absolute time because it has + better predictability and consistency than a periodic invocation, which + would rely on when the master node were elected or restarted. + + Relates to #43663 + +commit f35fcf7204bd9735d6cb688a0956455a915d19ef +Author: Lisa Cawley +Date: Fri Oct 4 13:11:05 2019 -0700 + + [DOCS] Adds security content in the Elasticsearch Reference (#47596) + +commit 45f12d18fb281e9f6df3827e2d612645873b494b +Author: James Rodewig +Date: Fri Oct 4 14:00:18 2019 -0400 + + [DOCS] Reformat shrink index API docs (#46711) (#47586) + +commit 63b169b600886a2b049e681a13cc3d25c03b7c90 +Author: Martijn van Groningen +Date: Fri Oct 4 12:54:49 2019 -0500 + + Upgrade joni from 2.1.6 to 2.1.29 (#47570) + + Backport of #47374 + + Changed the Grok class to use searchInterruptible(...) instead of search(...) + otherwise we can't interrupt long running matching via the thread watch + dog. + + Joni now also provides another way to interrupt long running matches. + By invoking the interrupt() method on the Matcher. We need then to refactor + the watch thread dog to keep track of Matchers instead of Threads, but + it is a better way of doing this, since interrupting would be more direct + (not every 30k iterations) and efficient (checking a volatile field). + This work needs to be done in a follow up. + +commit 0b4fb05540cb32fa2ed3099ed05b2b26a4257379 +Author: James Rodewig +Date: Fri Oct 4 13:50:09 2019 -0400 + + [DOCS] Reformat refresh API docs (#46667) (#47589) + +commit a66c0dcd957c9b0a132e12425cf37ad88326203d +Author: James Baiera +Date: Tue Sep 24 15:16:40 2019 -0400 + + Add pipeline to ensure unique Enrich index documents (#46348) + + Adds a pipeline that removes ids and routing from documents before indexing + them into enrich indices. Enrich documents may come from multiple indices, + and thus have id collisions on them. This pipeline ensures that documents + with colliding id fields do not clobber one another during the reindex operation + while executing an enrich policy. + +commit ee952da2e26ddb0efc20b87c50fc9ab8eaf1903b +Author: Przemysław Witek +Date: Fri Oct 4 17:54:51 2019 +0200 + + [7.x] Implement evaluation API for multiclass classification problem (#47126) (#47343) + +commit e3aab1295e429f99139dab3a2c8c67f03c0a4226 +Author: Jack Conradson +Date: Fri Oct 4 08:36:44 2019 -0700 + + Add a ScriptRoot to consolidate global data necessary for multiple passes (#47532) + + This PR is to get plumbing in for a ScriptRoot class that will consolidate + several pieces of state required by potentially multiple passes including + PainlessLookup, CompilerSettings, FunctionTable, the root class node, and a + synthetic counter. It's possible more may be added to this as we move + forward and slowly make the the nodes have less mutable state. + +commit 9b3e5409c1e625d2bd398a3e181a06f9c863becb +Author: Lisa Cawley +Date: Fri Oct 4 08:19:10 2019 -0700 + + [7.x][DOCS] Copies security source files from stack-docs (#47534) + +commit 3cc8081274e9595262d3abb7d1ca564d1697b600 +Author: James Rodewig +Date: Fri Oct 4 11:02:50 2019 -0400 + + [DOCS] Correct headings for split index API docs + +commit bb82addf35dafdd4f58c5bf89d39bc7617fcc88f +Author: James Rodewig +Date: Fri Oct 4 10:41:31 2019 -0400 + + [DOCS] Reformat split index API docs (#46713) (#47578) + +commit a46f312dedd1900360f481d06ae436d22d26e945 +Author: Andrei Stefan +Date: Fri Oct 4 16:05:55 2019 +0300 + + SQL: fix multi full-text functions usage with aggregate functions (#47444) + + * Skip functions involving full-text predicates when replacing multiple + aggregate functions with "stats" or "matrix_stats" aggregations. + + (cherry picked from commit bb14ba83128dfb7a70f825ea08b1524072fb9ad0) + +commit 2b16d7bcf8932bf804fed4e7462369b7e443e88c +Author: Alpar Torok +Date: Fri Oct 4 16:12:53 2019 +0300 + + Backport testclusters all (#47565) + + * Bwc testclusters all (#46265) + + Convert all bwc projects to testclusters + + * Fix bwc versions config + + * WIP fix rolling upgrade + + * Fix bwc tests on old versions + + * Fix rolling upgrade + +commit 8c180a77f0dfbc1ab1ddea54a61ae115d86a9647 +Author: Przemysław Witek +Date: Fri Oct 4 15:12:18 2019 +0200 + + [7.x] Fix serialization of evaluation response. (#47557) (#47566) + +commit c1be7a802c8a9426ccd4364a7762bcd90b3b527d +Author: Armin Braun +Date: Fri Oct 4 13:55:16 2019 +0200 + + Simplify Snapshot Delete Process (#47439) (#47533) + + We don't need to read the SnapshotInfo for a + snapshot to determine the indices that need to + be updated when it is deleted as the `RepositoryData` + contains that information already. + This PR makes it so the `RepositoryData` is used to + determine which indices to update and also removes + the special handling for deleting snapshot metadata + and the CS snapshot blob and has those simply be + deleted as part of the deleting of other unreferenced + blobs in the last step of the delete. + + This makes the snapshot delete a little faster and + more resilient by removing two RPC calls + (the separate delete and the get). + + Also, this shortens the diff with #46250 as a + side-effect. + +commit ec9b77deaaff3efa9a1d653bac4f769a1c5541c1 +Author: Przemysław Witek +Date: Fri Oct 4 13:47:19 2019 +0200 + + [7.x] Implement new analysis type: classification (#46537) (#47559) + +commit 65c473bd4bffea9691a2c01ab5d6cdf42a648279 +Author: Alpar Torok +Date: Fri Oct 4 14:02:24 2019 +0300 + + Fix windows packaging tests (#47554) + + On windows, it happens that the process we called terminates but some + other process it creates still has the same output strems and thus the + files open, so we can't clean it up. + + This PR makes the cleanup a best effort. + +commit 31a5e1c7eefaae8fb2bc48ce8564ba15f3aa9c0d +Author: David Roberts +Date: Fri Oct 4 09:16:56 2019 +0100 + + [ML] More accurate job memory overhead (#47516) + + When an ML job runs the memory required can be + broken down into: + + 1. Memory required to load the executable code + 2. Instrumented model memory + 3. Other memory used by the job's main process or + ancilliary processes that is not instrumented + + Previously we added a simple fixed overhead to + account for 1 and 3. This was 100MB for anomaly + detection jobs (large because of the completely + uninstrumented categorization function and + normalize process), and 20MB for data frame + analytics jobs. + + However, this was an oversimplification because + the executable code only needs to be loaded once + per machine. Also the 100MB overhead for anomaly + detection jobs was probably too high in most cases + because categorization and normalization don't use + _that_ much memory. + + This PR therefore changes the calculation of memory + requirements as follows: + + 1. A per-node overhead of 30MB for _only_ the first + job of any type to be run on a given node - this + is to account for loading the executable code + 2. The established model memory (if applicable) or + model memory limit of the job + 3. A per-job overhead of 10MB for anomaly detection + jobs and 5MB for data frame analytics jobs, to + account for the uninstrumented memory usage + + This change will enable more jobs to be run on the + same node. It will be particularly beneficial when + there are a large number of small jobs. It will + have less of an effect when there are a small number + of large jobs. + +commit defc97a3009f5e716e158bf6bbec25c3df2772c4 +Author: David Roberts +Date: Fri Oct 4 09:16:10 2019 +0100 + + Remove fallback for controller location (#47104) + + This change removes the temporary controller + location fallback introduced in #47013. + + Relates elastic/ml-cpp#593 + +commit a57cb5843f9185fe5a4f51f5af0acce77cd24890 +Author: István Zoltán Szabó +Date: Fri Oct 4 08:42:30 2019 +0200 + + [DOCS] Fixes an attribute in the update datafeed API docs. (#47551) + +commit 67bf3a4f51528592f2b56c6eb05df2c0883a7cea +Author: Alpar Torok +Date: Fri Oct 4 09:44:17 2019 +0300 + + Fix default distro bwc tests + +commit 97a0b7dcbc7d46cdd73b6719ef8d1e3af3954cdc +Author: Alpar Torok +Date: Fri Oct 4 08:41:06 2019 +0300 + + Make All OS tests run on GCP instances (#46924) + + This PR makes the necesary adaptations to the tests and adds a power shell script to + invoke the OS tests on GCP instances connected as CI workers. + + Also noticed that logs were not being produced by the tests and that theses were not using log4j so fixed that too. + + One of the difficulties in working on theses tests was that the tests just stalled with no indication where the problem is. + To ease with the debugging, after process explorer suggested that the tests are running some commands, we now have multiple timeouts: one for the tests ( which will generate a thread dump ) and one for individual commands ( that bails with the command being ran and output and error so far ) to make it easier to see what went wrong. + + The tests were blocking because apparently the pipes to the sub-process were not closing, thus the threads were blocking on them and we were blocking indefinitely on the join. I'm not sure why this doesn't happen in vagrant, but we now properly deal with it. + +commit f32692208e8875a4e301072f5d0191373b87ba12 +Author: Ryan Ernst +Date: Thu Oct 3 21:05:05 2019 -0700 + + Add explanations to script score queries (#46693) (#47548) + + While function scores using scripts do allow explanations, they are only + creatable with an expert plugin. This commit improves the situation for + the newer script score query by adding the ability to set the + explanation from the script itself. + + To set the explanation, a user would check for `explanation != null` to + indicate an explanation is needed, and then call + `explanation.set("some description")`. + +commit d371f9d44d6b5d9b2c123176a22ecd4a3b2b62c0 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Oct 4 13:17:52 2019 +1000 + + Fix for ApiKeyIntegTests related to Expired API keys remover (#43477) (#47546) + + When API key is invalidated we do two things first it tries to trigger `ExpiredApiKeysRemover` task + and second, we do index the invalidation for the API key. The index invalidation may happen + before the `ExpiredApiKeysRemover` task is run and in that case, the API key + invalidated will also get deleted. If the `ExpiredApiKeysRemover` runs before the + API key invalidation is indexed then the API key is not deleted and will be + deleted in the future run. + This behavior was not captured in the tests related to `ExpiredApiKeysRemover` + causing intermittent failures. + This commit fixes those tests by checking if the API key invalidated is reported + back when we get API keys after invalidation and perform the checks based on that. + + Closes #41747 + +commit 5e4732f2bbb388b84e6f0b8faaea7965ed7206ca +Author: Nhat Nguyen +Date: Thu Oct 3 10:34:15 2019 -0400 + + Limit number of retaining translog files for peer recovery (#47414) + + Today we control the extra translog (when soft-deletes is disabled) for + peer recoveries by size and age. If users manually (force) flush many + times within a short period, we can keep many small (or empty) translog + files as neither the size or age condition is reached. We can protect + the cluster from running out of the file descriptors in such a situation + by limiting the number of retaining translog files. + +commit f81d9a4aa6c35b7f21cb7bc310ca49b4e5e26d2c +Author: Lisa Cawley +Date: Thu Oct 3 17:06:26 2019 -0700 + + Update link to ls monitoring settings (#47529) (#47542) + +commit b8e7f5069d6014b7bea8a96f6afb6e01fd9a0944 +Author: James Rodewig +Date: Thu Oct 3 16:43:26 2019 -0400 + + [DOCS] Reformat index recovery API docs (#46493) (#47527) + +commit 56e7668c784229bc202a814a12580755a5688fd4 +Author: James Rodewig +Date: Thu Oct 3 15:57:11 2019 -0400 + + [DOCS] Reformat index shard stores API docs (#46504) + +commit dc02868671c2ce3ba5a1b6e9a9e1ccf59919f944 +Author: Francois-Clement Brossard +Date: Fri Oct 4 04:22:48 2019 +0900 + + [DOCS] Update painless statements with if/else example (#47485) + +commit 392013cf057bb262aee2e1b94472406109688e12 +Author: Alpar Torok +Date: Thu Oct 3 18:12:22 2019 +0300 + + Fix double start of test cluster + +commit 9c7b58900cf5ebd91447bef69a706fd29516dd7f +Author: Lisa Cawley +Date: Thu Oct 3 07:41:40 2019 -0700 + + [DOCS] Fixes missing link title (#47481) + +commit fd6a585009b4f9cff7b20b030ee453c294a4e6b9 +Author: Ioannis Kakavas +Date: Thu Oct 3 17:14:26 2019 +0300 + + Fix ADRealmTests in FIPS 140 JVMs (#47437) (#47506) + + The changes introduced in #47179 made it so that we could try to + build an SSLContext with verification mode set to None, which is + not allowed in FIPS 140 JVMs. This commit address that + +commit 510d7d84b2f4f22d47e7d6f42ecba9c7e371c713 +Author: Alpar Torok +Date: Thu Oct 3 16:31:22 2019 +0300 + + bwc tests: make sure first node is only started once + +commit edbc375916821451cf916bc3ed31b4ec14e5ad43 +Author: Paweł Krześniak +Date: Thu Oct 3 15:25:35 2019 +0200 + + [DOCS] Change index name in rollover ILM example (#47492) + + The warning section above the example tells that index name has to end with the digits but the example itself uses index name without digits which is confusing. + +commit 4a516f4ae2788e4b9dbeb5f7d994bcabb199f470 +Author: AndyHunt66 +Date: Thu Oct 3 13:40:31 2019 +0100 + + [DOCS] Remove duplicated half-sentence from secure settings docs (#47498) + +commit 93032c9dea3fe3937093a8f2663702e6a6135728 +Author: Alpar Torok +Date: Thu Oct 3 15:22:01 2019 +0300 + + Fix bwcTests for versions < 6.5.0 + +commit 9dcdbea22817ec822663302bd40233590d31d18d +Author: István Zoltán Szabó +Date: Thu Oct 3 13:12:19 2019 +0200 + + [DOCS] Amends update datafeed API docs (#47448) + +commit bac119f672da79c8aad27c01e36082646e740c6a +Author: Armin Braun +Date: Thu Oct 3 12:47:50 2019 +0200 + + Fix getSnapshotIndexMetaData Exception Behavior (#47488) (#47496) + + If we fail to read the global metadata in a snapshot + we would throw `SnapshotMissingException` but wouldn't + do so for the index metadata. + This is breaking SLM tests at a low rate because they + use `SnapshotMissingException` thrown from snapshot status APIs + to wait for a snapshot being gone. + Also, we should be consistent here in general and not leak the + `NoSuchFileException` to the transport layer for index meta. + + Closes #46508 + +commit 7549be44891dcecbd8e618a6da35227701bcfa40 +Author: Armin Braun +Date: Thu Oct 3 11:10:48 2019 +0200 + + Fix es.http.cname_in_publish_address Deprecation Logging (#47451) + + Since the property defaulted to `true` this deprecation logging + runs every time unless its set to `false` manually (in which case + it should've also logged but didn't). + I didn't add a tests and removed the tests we had in `7.x` that + covered this logging. I did move the check out of the `if (InetAddresses.isInetAddress(hostString) == false) {` condition so this is sort-of covered by the REST tests. + IMO, any unit-test of this would be somewhat redundant and would've forced + adding a field that just indicates that the deprecated property was used to + every instance which seemed pointless. + + Closes #47436 + +commit 3bb17a0c28159a7bd7d86d1918f184a9878571f6 +Author: Russ Cam +Date: Thu Oct 3 19:00:33 2019 +1000 + + Remove string interning from release notes (#47489) + + This commit removes the string interning issue and PR from the 7.4.0 release notes. + + The PR https://github.com/elastic/elasticsearch/pull/41039 was closed and not merged. + +commit 0a14bb174fa292c437cdcb20bacbc80bd3adf54a +Author: Alpar Torok +Date: Thu Oct 3 10:50:46 2019 +0300 + + Remove eclipse conditionals (#44075) + + * Remove eclipse conditionals + + We used to have some meta projects with a `-test` prefix because + historically eclipse could not distinguish between test and main + source-sets and could only use a single classpath. + This is no longer the case for the past few Eclipse versions. + + This PR adds the necessary configuration to correctly categorize source + folders and libraries. + With this change eclipse can import projects, and the visibility rules + are correct e.x. auto compete doesn't offer classes from test code or + `testCompile` dependencies when editing classes in `main`. + + Unfortunately the cyclic dependency detection in Eclipse doesn't seem to + take the difference between test and non test source sets into account, + but since we are checking this in Gradle anyhow, it's safe to set to + `warning` in the settings. Unfortunately there is no setting to ignore + it. + + This might cause problems when building since Eclipse will probably not + know the right order to build things in so more wirk might be necesarry. + +commit d55378e8e7920f672d57f1f027df95a0eb2f77ee +Author: Ioannis Kakavas +Date: Thu Oct 3 11:03:29 2019 +0300 + + Add realm uniqueness enforcement in 7.4 migration docs (#47487) + + Add the change to the migration docs as it was omitted in #46253 + +commit 5a3fa4a479dd9a7d77172da34eaeef49a1403467 +Author: Jim Ferenczi +Date: Thu Oct 3 01:23:45 2019 +0200 + + Add client jar for mapper-extras (#47430) + + The rest high level client has a dependency on mapper-extras but the jar + is not published so this commit adds a client jar for this module. + + Closes #47413 + +commit 44fdf2020aa762d985f739cb8f543860b08ec80b +Author: Nhat Nguyen +Date: Wed Oct 2 17:30:55 2019 -0400 + + Always flush in FullClusterRestartIT#testRecovery (#47465) + + The pattern in the latest failure is similar to the source fixed in #46956 + but relates to synced-flush. If peer recovery happens after indexing, + and indexing flushes some shard at the end, then a synced flush in the + test will not roll or commit translog. + + Closes #46712 + +commit 0beb5263b4ae363c453c0a7ba17063125180bdb9 +Author: Armin Braun +Date: Wed Oct 2 23:26:18 2019 +0200 + + Fix Snapshot Finalization not Waiting for Index Metadata (#47445) (#47459) + + * Fix Snapshot Finalization not Waiting for Index Metadata + + We were mixing up the listeners here which led to the final listener + that should be called after all the metadata has been written + to be called before that. + I fixed this by removing the one redundant listener and flattening + the logic out. + + * Closes #47425 + +commit 6b96f53ea05d9f05493d027980bda7a34721d348 +Author: James Rodewig +Date: Wed Oct 2 16:57:11 2019 -0400 + + [DOCS] Change xrefs to external links in 7.4 rls highlights (#47474) + +commit 52b97ec5398055cd60ada4e61b85777c18c2be83 +Author: Jason Tedor +Date: Wed Oct 2 15:11:51 2019 -0500 + + Allow setting validation against arbitrary types (#47264) + + Today when settings validate, they can only validate against settings + that are of the same type. While this strong-type is convenient from a + development perspective, it is too limiting in that some settings need + to validate against settings of a different type. For example, the list + setting xpack.monitoring.exporters..host wants to validate + that it is non-empty if and only if the string setting + xpack.monitoring.exporters..type is "http". Today this is + impossible since the settings validation framework only allows that + setting to validate against other list settings. This commit increases + the flexibility here to validate against settings of arbitrary type, at + the expense of losing strong-typing during development. + +commit 2e3eb4b24e3a86ebbd1eb83dad2bff6589ab2533 +Author: Lee Hinman +Date: Wed Oct 2 12:29:04 2019 -0600 + + Add API to execute SLM retention on-demand (#47405) (#47463) + + * Add API to execute SLM retention on-demand (#47405) + + This is a backport of #47405 + + This commit adds the `/_slm/_execute_retention` API endpoint. This + endpoint kicks off SLM retention and then returns immediately. + + This in particular allows us to run retention without scheduling it + (for entirely manual invocation) or perform a one-off cleanup. + + This commit also includes HLRC for the new API, and fixes an issue + in SLMSnapshotBlockingIntegTests where retention invoked prior to the + test completing could resurrect an index the internal test cluster + cleanup had already deleted. + + Resolves #46508 + Relates to #43663 + +commit 6acc5ca8d1d13216f13f29d7342d43b4447b12a2 +Author: Mark Vieira +Date: Wed Oct 2 11:04:48 2019 -0700 + + Remove groovy test code from buildSrc (#47416) + +commit 013d87d7167cdaae9560cd925ac6d981aff18845 +Author: Lee Hinman +Date: Wed Oct 2 10:01:05 2019 -0600 + + Fix AllocationRoutedStepTests.testConditionMetOnlyOneCopyAlloc… (#47313) + + * Fix AllocationRoutedStepTests.testConditionMetOnlyOneCopyAllocated + + These tests were using randomly generated includes/excludes/requires for + routing, however, it was possible to generate mutually exclusive + allocation settings (about 1 out of 50,000 times for my runs). + + This splits the test into three different tests, and removes the + randomization (it doesn't add anything to the testing here) to fix the + issue. + + Resolves #47142 + +commit c340814b344523647aa700a49158ac0dc947e06f +Author: Jim Ferenczi +Date: Wed Oct 2 15:23:39 2019 +0200 + + Fix highlighting of overlapping terms in the unified highlighter (#47227) + + The passage formatter that the unified highlighter use doesn't handle terms with overlapping offsets. + For tokenizer that provides multiple segmentation of the same terms (edge ngram for instance) the formatter + should select the largest span in order to highlight the term only once. This change implements this logic. + +commit 4f722f0f538931458cd061de777500525ad64a19 +Author: Ioannis Kakavas +Date: Wed Oct 2 17:18:12 2019 +0300 + + Fix Active Directory tests (#47358) (#47440) + + Fixes multiple Active Directory related tests that run against the + samba fixture. Some were failing since we changed the realm settings + format in 7.0 and a few were slightly broken in other ways. + We can move to cleanup the tests in a follow up but this work fits + better to be done with or after we move the tests from a Samba + based fixture to a real(-ish) Microsoft Active Directory based + fixture. + + Resolves: #33425, #35738 + +commit 2228a7dd8d073e5996c4abb93d228c041acf8a63 +Author: Benjamin Trent +Date: Wed Oct 2 09:49:46 2019 -0400 + + [ML][Inference] adding ensemble model objects (#47241) (#47438) + + * [ML][Inference] adding ensemble model objects + + * addressing PR comments + + * Update TreeTests.java + + * addressing PR comments + + * fixing test + +commit b9541eb3afbf549903cd3c1c2c4b48c324eea744 +Author: Dimitris Athanasiou +Date: Wed Oct 2 16:24:21 2019 +0300 + + [7.x][ML] Make PUT data frame analytics action a master node action (… (#47433) + + While it seemed like the PUT data frame analytics action did not + have to be a master node action as the config is stored in an index + rather than the cluster state, there are other subtle nuances which + make it worthwhile to convert it. In particular, it helps maintain + order of execution for put actions which are anyhow user driven and + are expected to have low volume. + + This commit converts `TransportPutDataFrameAnalyticsAction` from + a handled transport action to a master node action. + + Note this means that the action might fail in a mixed cluster + but as the API is still experimental and not widely used there will + be few moments more suitable to make this change than now. + +commit f7980e97451a8d619bba7830d2d381634b7952ac +Author: Yannick Welsch +Date: Wed Oct 2 14:25:01 2019 +0200 + + Adapt version constants after backport (#47353) + +commit 99d2fe295dae3409ccc4c208375942473a4abd54 +Author: Yannick Welsch +Date: Wed Oct 2 11:04:19 2019 +0200 + + Use optype CREATE for single auto-id index requests (#47353) + + Changes auto-id index requests to use optype CREATE, making it compliant with our docs. + This will also make these auto-id index requests compatible with the new "create-doc" index + privilege (which is based on the optype), the default optype is changed to create, just as it is + already documented. + +commit 0024695dd8c28256deaae60d3f3b0d79c86ecac8 +Author: Yannick Welsch +Date: Wed Oct 2 11:02:53 2019 +0200 + + Disallow externally generated autoGeneratedTimestamp (#47341) + + The autoGeneratedTimestamp field is internally used to speed up indexing of operations with + auto-ids, as we can rule out duplicates. Setting this field externally can make the index + inconsistent, resulting in duplicate documents with same id. + +commit 8c11fe610efd4118d4786e5784c5316898b98899 +Author: Yannick Welsch +Date: Tue Oct 1 14:53:27 2019 +0200 + + Use standard semantics for retried auto-id requests (#47311) + + Adds support for handling auto-id requests with optype CREATE. Also simplifies the code + handling this by using the standard indexing path when dealing with possible retry conflicts. + + Relates #47169 + +commit 7b2613db55923378eb2bc3663e20d19330e4eeab +Author: Yannick Welsch +Date: Mon Sep 30 16:54:46 2019 +0200 + + Allow optype CREATE for append-only indexing operations (#47169) + + Bulk requests currently do not allow adding "create" actions with auto-generated IDs. + This commit allows using the optype CREATE for append-only indexing operations. This is + mainly the user facing aspect of it. + +commit a032f9b2d544cba9a4c3c4fc6c7bf24b74284b95 +Author: Alpar Torok +Date: Wed Oct 2 14:37:00 2019 +0300 + + Backport testclusters fix bwc (#47363) + + * Add support for bwc for testclusters and convert full cluster restart (#45374) + + * Testclusters fix bwc (#46740) + + Additions to make testclsuters work with lather versions of ES + + * Do common node config on bwc tests + + Before this PR we always ever ran `ElasticsearchCluster.start` once, and + the common node config was never done. + This becomes apparent in upgrading from `6.x` to `7.x` as the new config + is missing preventing the cluster from starting. + + * Do common node config on bwc tests + + Before this PR we always ever ran `ElasticsearchCluster.start` once, and + the common node config was never done. + This becomes apparent in upgrading from `6.x` to `7.x` as the new config + is missing preventing the cluster from starting. + + * Fix logic to pick up snapshot from 6.x + + * Make sure ports are cleared + + * Fix test + + * Don't clear all the config as we rely on it + + * Fix removal of keys + +commit 42453aec960a01e9edd7df85997e0bac3c1c7abf +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Oct 2 12:35:28 2019 +0200 + + Fix XPackPlugin usages in tests (#47252) + + XPackPlugin holds data in statics and can only be initialized once. This + caused tests to fail primarily when running with a low max-workers. + + Replaced usages with the LocalStateCompositeXPackPlugin, which handles + this properly for testing. + +commit b5a2afccb261fe99da36e42ebf2581ac919d71dc +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Oct 2 12:32:46 2019 +0200 + + MockSearchService concurrency fix (#47139) + + Fixed MockSearchService concurrency, assertNoInFlightContext could + have false negative result (rarely). + + Split out from #46060 + Closes #47048 + +commit 697c693ee7bb4f6ff03415603f0c24a42c9d58e8 +Author: Alan Woodward +Date: Wed Oct 2 11:19:25 2019 +0100 + + Reset Token position on reuse in scripted analysis (#47424) + + Most of the information in AnalysisPredicateScript.Token is pulled directly + from its underlying AttributeSource, but we also keep track of the token position, + and this state is held directly on the Token. This information needs to be reset when + the containing ScriptFilteringTokenFilter or ScriptedConditionTokenFilter is re-used. + + Fixes #47197 + +commit 4379a3c52b79f66d87472df9f804eddbffae10dc +Author: David Roberts +Date: Wed Oct 2 08:58:56 2019 +0100 + + [ML] Throttle the delete-by-query of expired results (#47177) + + Due to #47003 many clusters will have built up a + large backlog of expired results. On upgrading to + a version where that bug is fixed users could find + that the first ML daily maintenance task deletes + a very large amount of documents. + + This change introduces throttling to the + delete-by-query that the ML daily maintenance uses + to delete expired results to limit it to deleting an + average 200 documents per second. (There is no + throttling for state/forecast documents as these + are expected to be lower volume.) + + Additionally a rough time limit of 8 hours is applied + to the whole delete expired data action. (This is only + rough as it won't stop part way through a single + operation - it only checks the timeout between + operations.) + + Relates #47103 + +commit 42c5054e52ae406f0da56d620e01d0d8ea60ba50 +Author: Jim Ferenczi +Date: Wed Oct 2 09:54:10 2019 +0200 + + Fix alias field resolution in match query (#47369) + + Synonym queries (when two tokens/paths start at the same position) use the alias field instead + of the concrete field to build Lucene queries. This commit fixes this bug by resolving the alias field upfront in order to provide the concrete field to the actual query parser. + +commit 033aa9cf9b2939db041a3ece9e32da9e89ef402b +Author: István Zoltán Szabó +Date: Wed Oct 2 10:26:20 2019 +0200 + + [DOCS] Adds examples to the PUT dfa and the evaluate dfa APIs (#46966) + + * [DOCS] Adds examples to the PUT dfa and the evaluate dfa APIs. + + * [DOCS] Removes extra lines from examples. + + * Update docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc + + Co-Authored-By: Lisa Cawley + + * Update docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc + + Co-Authored-By: Lisa Cawley + + * [DOCS] Explains examples. + +commit 7739938930fa1e1cd7814c7cf930b6121716c489 +Author: David Turner +Date: Wed Oct 2 09:27:17 2019 +0100 + + Clarify that you cannot abort an upgrade (#47342) + + We do mention that rolling back an upgrade requires a restore from a snapshot, + but it's hidden at the bottom of the "preparing to upgrade" instructions on a + different page from the actual upgrade instructions. This commit duplicates the + preparatory instructions onto the pages containing the actual upgrade + instructions and rewords the point about rollbacks a bit. + +commit 98c1c2f650c0b046c2c120994c076f7126c79b6c +Author: Robin Clarke +Date: Wed Oct 2 10:22:02 2019 +0200 + + Clearer language around upgrade sequence (#47422) + +commit 1cecbd0cd3a5b0a6e066898f7d157fd42085c017 +Author: István Zoltán Szabó +Date: Wed Oct 2 10:04:35 2019 +0200 + + [DOCS] Fine tunes update anomaly detection job API documentation (#47280) + + * [DOCS] Fine tunes update anomaly detection job API documentation. + * [DOCS] Removes delimiter to fix the table. + +commit 6a9f04ee760cfe901772777e0bdd90e76dfd470c +Author: István Zoltán Szabó +Date: Wed Oct 2 09:49:59 2019 +0200 + + [DOCS] Fixes typos in the PUT dfa and the evaluate dfa documentation. (#47348) + +commit 36884a3c32a8ec63a4c9b14d8ac0241cc7b2180d +Author: Dimitris Athanasiou +Date: Wed Oct 2 10:24:05 2019 +0300 + + [7.x][ML] Restore analytics state if available (#47128) (#47393) + + This commit restores the model state if available in data + frame analytics jobs. + + In addition, this changes the start API so that a stopped job + can be restarted. As we now store the progress in the state index + when the task is stopped, we can use it to determine what state + the job was in when it got stopped. + + Note that in order to be able to distinguish between a job + that runs for the first time and another that is restarting, + we ensure reindexing progress is reported to be at least 1 + for a running task. + +commit bd5f64848e88f8be3948a06bd09756fd5276e256 +Author: Ryan Ernst +Date: Tue Oct 1 22:09:38 2019 -0700 + + Clarify missing java error message (#46160) + + Since the bundled jdk was added to Elasticsearch, there are now 2 ways + java can be missing. Either JAVA_HOME is set but does not exist, or the + bundled jdk does not exist. This commit improves the error messages in + those two cases, and also ensures our tests cover both cases. + +commit 5cfcd7c4587eb9ed9ad603fbc76730a9d0b31801 +Author: Nhat Nguyen +Date: Fri Sep 27 22:21:20 2019 -0400 + + Re-fetch shard info of primary when new node joins (#47035) + + Today, we don't clear the shard info of the primary shard when a new + node joins; then we might risk of making replica allocation decisions + based on the stale information of the primary. The serious problem is + that we can cancel the current recovery which is more advanced than the + copy on the new node due to the old info we have from the primary. + + With this change, we ensure the shard info from the primary is not older + than any node when allocating replicas. + + Relates #46959 + + This work was done by Henning in #42518. + + Co-authored-by: Henning Andersen + +commit ff15495b9835a443cd6d536e6ce489af9f7de6cd +Author: Mark Vieira +Date: Tue Oct 1 16:33:58 2019 -0700 + + Remove empty buildSrc subproject (#47415) + +commit 079bf887c00e4dcdba75b4256bbebf5c508f6b25 +Author: James Rodewig +Date: Tue Oct 1 17:07:28 2019 -0400 + + [DOCS] Reorder index APIs alphabetically (#46981) (#47402) + +commit ba6ee2d40db09338759d80f2d25ccfbbbc5707b8 +Author: Gordon Brown +Date: Tue Oct 1 14:53:10 2019 -0600 + + [7.x] Adjust randomization in cluster shard limit tests (#47254) + + This commit adjusts randomization for the cluster shard limit tests so + that there is often more of a gap left between the limit and the size of + the first index. This allows the same randomization to be used for all + tests, and alleviates flakiness in + `testIndexCreationOverLimitFromTemplate`. + +commit 99b25d37409048a73b317e6ad98bcfc1c6821f45 +Author: David Turner +Date: Tue Oct 1 19:57:25 2019 +0100 + + Keep nodes above watermark in testAutomaticReleaseOfIndexBlock (#47387) + + Today the comment boldly claims that this line of code keeps nodes above the + 10-byte low watermark when in fact this is not true at all. This change fixes + this so that it really does keep nodes above the low watermark. + + Fixes #45338. Again. + +commit 0179f93544f6e10f215142f572c3e4cebed14029 +Author: James Rodewig +Date: Tue Oct 1 14:49:14 2019 -0400 + + [DOCS] Reformat simulate pipeline API (#47301) (#47398) + +commit aeb4edce3af07a3bbbc0a86272f9da0cf2a076fd +Author: James Rodewig +Date: Tue Oct 1 14:48:18 2019 -0400 + + [DOCS] Reformat put pipeline API (#47171) (#47395) + +commit f5fe5e7cd61a2ad57f36bb308d1737c31f2f789e +Author: Benjamin Trent +Date: Tue Oct 1 13:31:25 2019 -0400 + + [7.x] [ML][Inference] Adding preprocessors to definition object (#47320) (#47370) + + * [ML][Inference] Adding preprocessors to definition object (#47320) + + * [ML][Inference] Adding preprocessors to definition object + + * Update TrainedModelConfig.java + + * adjusting for backport + +commit 66116e39baf13f225187ff6815244b499a4229d5 +Author: lcawl +Date: Tue Oct 1 10:09:29 2019 -0700 + + [DOCS] Edits ML release notes + +commit 3d6ef6a90e885fdf0caefcbffc6d9b2b8cdab839 +Author: Armin Braun +Date: Tue Oct 1 19:05:43 2019 +0200 + + Speed up and Reorder Snapshot Delete Operations (#47293) (#47350) + + This is a preliminary of #46250 making the snapshot + delete work by doing all the metadata updates first + and then bulk deleting all of the now unreferenced + blobs. + Before this change, the metadata updates for each shard + and subsequent deletion of the blobs that have become unreferenced + due to the delete would happen sequentially shard-by-shard + parallelising only over all the indices in the snapshot. + This change makes it so the all the metadata updates + happen in parallel on a shard level first. + Once all of the updates of shard-level metadata have finished, + all the now unreferenced blobs are deleted in bulk. + This has two benefits (outside of making #46250 a smaller change): + * We have a lower likelihood of failing to update shard level metadata because + it happens with priority and a higher degree of parallelism + * Deleting of unreferenced data in the shards should go much faster in many cases (rolling indices, large number of indices with many unchanged shards) as well because a number of small bulk deletions (just two blobs for `index-N` and `snap-` for each unchanged shard) are grouped into larger bulk deletes of `100-1000` blobs depending on Cloud provider (even though the final bulk deletes are happening sequentially this should be much faster in almost all cases as you'd parallelism of 50 (GCS) to 500 (S3) snapshot threads to achieve the same delete rates when deleting from unchanged shards). + +commit 0e1b77568ae6fa29cf8cfdfb1552ecdd89b9ff0f +Author: Michael Basnight +Date: Tue Oct 1 10:24:23 2019 -0500 + + Add enable checks to missing enrich plugin methods (#47187) + + Some of the server side objects that do not need to be created unless + enrich is enabled were still being created. This commit fixes that. + +commit e70220857db4a7fd574f64be946f95baec8eed67 +Author: James Rodewig +Date: Tue Oct 1 12:22:50 2019 -0400 + + [DOCS] Document cat tasks API (#47321) (#47375) + +commit c93b39c65b0729c00fe7e683c4c905e50ca7c239 +Author: Colin Goodheart-Smithe +Date: Tue Oct 1 13:09:04 2019 +0100 + + Adds version 7.4.1 + +commit 5bdf25320a7eb2eb0247b3c6c94ba8934ad2d861 +Author: Mark Tozzi +Date: Tue Oct 1 10:58:44 2019 -0400 + + Documentation notes for Range field histograms (#46890) (#47366) + +commit 5ba543fd6c4d2b86aabdf067b20fe9151372393f +Author: Lisa Cawley +Date: Tue Oct 1 06:00:26 2019 -0700 + + [DOCS] Adds machine learning PRs to release notes (#47316) + +commit 2ca075dee4f1ab25a59cc00718cbd547db174e46 +Author: James Rodewig +Date: Tue Oct 1 09:10:20 2019 -0400 + + [DOCS] Remove coming tags for 7.4.0 release (#47318) + +commit 78558a7b2f37aebace2ca9c9e1bbd8d8d9112640 +Author: Albert Zaharovits +Date: Tue Oct 1 17:05:24 2019 +0300 + + Fix AD realm additional metadata (#47179) + + Due to a regression bug the metadata Active Directory realm + setting is ignored (it works correctly for the LDAP realm type). + This commit redresses it. + + Closes #45848 + +commit f792dbf23915d9c4b97c75964d41e7494354a53e +Author: Marios Trivyzas +Date: Tue Oct 1 15:59:25 2019 +0300 + + SQL: Implement DATE_PART function (#47206) + + DATE_PART(, ) is a function that allows + the user to extract the specified unit from a date/datetime field + similar to the EXTRACT ( FROM ) but + with different names and aliases for the units and it also provides more + options like `DATE_PART('tzoffset', datetimeField)`. + + Implemented following the SQL server's spec: https://docs.microsoft.com/en-us/sql/t-sql/functions/datepart-transact-sql?view=sql-server-2017 + with the difference that the argument is either a + literal single quoted string or gets a value from a table field, whereas + in SQL server keywords are used (unquoted identifiers) and it's not + possible to use a value coming for a table column. + + Closes: #46372 + (cherry picked from commit ead743d3579eb753fd314d4a58fae205e465d72e) + +commit 4335e077168ee844cd547afd9183ecd78518c2ad +Author: Benjamin Trent +Date: Tue Oct 1 08:20:33 2019 -0400 + + [7.x] [ML][Inference] adding .ml-inference* index and storage (#47267) (#47310) + + * [ML][Inference] adding .ml-inference* index and storage (#47267) + + * [ML][Inference] adding .ml-inference* index and storage + + * Addressing PR comments + + * Allowing null definition, adding validation tests for model config + + * fixing line length + + * adjusting for backport + +commit c43e932a0c7efec84219d9f0c3f5076647914c13 +Author: Tanguy Leroux +Date: Tue Oct 1 12:48:01 2019 +0200 + + Fix CharArraysTests.testConstantTimeEquals() (#47346) + + The change #47238 fixed a first issue (#47076) but introduced + another one that can be reproduced using: + + org.elasticsearch.common.CharArraysTests > testConstantTimeEquals FAILED + + java.lang.StringIndexOutOfBoundsException: String index out of range: 1 + at __randomizedtesting.SeedInfo.seed([DFCA64FE2C786BE3:ED987E883715C63B]:0) + at java.lang.String.substring(String.java:1963) + at org.elasticsearch.common.CharArraysTests.testConstantTimeEquals(CharArraysTests.java:74) + + REPRODUCE WITH: ./gradlew ':libs:elasticsearch-core:test' --tests + "org.elasticsearch.common.CharArraysTests.testConstantTimeEquals" + -Dtests.seed=DFCA64FE2C786BE3 -Dtests.security.manager=true -Dtests.locale=fr-CA + -Dtests.timezone=Pacific/Johnston -Dcompiler.java=12 -Druntime.java=8 + + that happens when the first randomized string has a length of 0. + +commit 3b06916fcd402161780842e225eabef38e60f7e8 +Author: Ioannis Kakavas +Date: Tue Oct 1 13:32:31 2019 +0300 + + Revert "Fix Active Directory tests (#47266)" + + This reverts commit 7d9c06421866843cf6b9c25065b591f0ba0a0cc9. + +commit a9cd42c05d95c44a7c3d73eaa8cb624cba60644b +Author: Howard +Date: Tue Oct 1 17:53:53 2019 +0800 + + Cancel recoveries even if all shards assigned (#46520) + + We cancel ongoing peer recoveries if a node joins the cluster with a completely + up-to-date copy of a shard, because we can use such a copy to recover a replica + instantly. However, today we only look for recoveries to cancel while there are + unassigned shards in the cluster. This means that we do not contemplate the + cancellation of the last few recoveries since recovering shards are not + unassigned. It might take much longer for these recoveries to complete than + would be necessary if they were cancelled. + + This commit fixes this by checking for cancellable recoveries even if all + shards are assigned. + +commit 03d717dc32b526c588a60cfa7bf12c1fb5880da4 +Author: Ignacio Vera +Date: Tue Oct 1 10:52:39 2019 +0200 + + Provide better error when updating geo_shape field mapper settings (#47281) (#47338) + +commit 7d9c06421866843cf6b9c25065b591f0ba0a0cc9 +Author: Ioannis Kakavas +Date: Tue Oct 1 10:48:30 2019 +0300 + + Fix Active Directory tests (#47266) + + Fixes multiple Active Directory related tests that run against the + samba fixture. Some were failing since we changed the realm settings + format in 7.0 and a few were slightly broken in other ways. + We can move to cleanup the tests in a follow up but this work fits + better to be done with or after we move the tests from a Samba + based fixture to a real(-ish) Microsoft Active Directory based + fixture. + + Resolves: #33425, #35738 + +commit f5c5411fe8dcfe4a099d0d43d816065ed80d01e3 +Author: Tanguy Leroux +Date: Tue Oct 1 08:39:55 2019 +0200 + + Differentiate base paths in repository integration tests (#47284) (#47300) + + This commit change the repositories base paths used in Azure/S3/GCS + integration tests so that they don't conflict with each other when tests + run in parallel on real storage services. + + Closes #47202 + +commit dd0af2e42577bcab0158193e8bb0afd8eee15ad9 +Author: Yannick Welsch +Date: Tue Oct 1 08:32:50 2019 +0200 + + Fix CloseIndexIT.testRelocatedClosedIndexIssue (#47169) + + Closes #47330 + +commit 33c5e5b09d472798cd9daaa3457af90a3353b520 +Author: Ioannis Kakavas +Date: Tue Oct 1 09:13:01 2019 +0300 + + Fix SSLErrorMessageTests in Windows (#47315) + + - Build paths with PathUtils#get instead of hard-coding a string with + forward slashes. + - Do not try to match the whole message that includes paths. The + file separator is `\\` in windows but when we throw an Elasticsearch + Exception, the message is formatted with LoggerMessageFormat#format + which replaces `\\` with `\` in Path names. That means that in Windows + the Exception message will contain paths with single backslashes while + the expected string that comes from Path#toString on filename and + env.configFile will contain double backslashes. There is no point in + attempting to match the whole message string for the purpose of this test. + + Resolves: #45598 + +commit fa0b1b641a589a723d87f6c3d298117d005aab3f +Author: Marios Trivyzas +Date: Tue Oct 1 09:11:34 2019 +0300 + + SQL: Add examples fo muting sql/csv integ tests (#47291) + + Add examples of failures for both sql and csv integeration + tests and instructions on how to mute them. + + (cherry picked from commit 591bba46516d770f5fc95a4c536dd7448b74dd49) + +commit 170b102ab5213d4c9342a36e1d59a28660e3817d +Author: István Zoltán Szabó +Date: Tue Oct 1 08:04:06 2019 +0200 + + [DOCS] Changes wording to move away from data frame terminology in the ES repo (#47093) + + * [DOCS] Changes wording to move away from data frame terminology in the ES repo. + Co-Authored-By: Lisa Cawley + +commit 45605cfd7a6dba39519f4c01d8b6c44ec617c029 +Author: Mark Vieira +Date: Mon Sep 30 15:05:07 2019 -0700 + + Use VAULT_TOKEN environment variable if it exists (#45525) + + + (cherry picked from commit b57f2ff68049a4d86ab37399531616b7ad26cfd9) + +commit f4e32a5f5f97d22e88e772afe27602ee2eff06c3 +Author: Alpar Torok +Date: Wed Sep 25 14:33:26 2019 +0300 + + Disable the use of artifactory in CI (#47100) + + + (cherry picked from commit cbe8b3645f74da0a7e2794536d84a3c46905e49e) + +commit 67f0ffd134524eab1ed5fc560c7e8b004058758b +Author: Ryan Ernst +Date: Mon Sep 30 10:52:10 2019 -0700 + + Ensure char array test uses different values (#47238) + + The test of constantTimeEquals could get unlucky and randomly produce + the same two strings. This commit tweaks the test to ensure the two + string are unique, and the loop inside constantTimeEquals is actually + executed (which requires the strings be of the same length). + + fixes #47076 + +commit 3d23cb44a37c9e1abc9ee24d3fc4eb2a62cd9eca +Author: Armin Braun +Date: Mon Sep 30 23:28:59 2019 +0200 + + Speed up Snapshot Finalization (#47283) (#47309) + + As a result of #45689 snapshot finalization started to + take significantly longer than before. This may be a + little unfortunate since it increases the likelihood + of failing to finalize after having written out all + the segment blobs. + This change parallelizes all the metadata writes that + can safely run in parallel in the finalization step to + speed the finalization step up again. Also, this will + generally speed up the snapshot process overall in case + of large number of indices. + + This is also a nice to have for #46250 since we add yet + another step (deleting of old index- blobs in the shards + to the finalization. + +commit bd2abeef40dc51074e481d6db352cf220dededa3 +Author: Marios Trivyzas +Date: Mon Sep 30 22:18:02 2019 +0300 + + SQL: [TESTS] Improve error messages on failures (#47308) + + When an integration test fails before the assertion of the results it's + missing information, like the file name and the line in the file where + the test resides. + + (cherry picked from commit 683dc7213311d13c81e06829e08f3f9f80ebf73a) + +commit 890951113f527dec9b38c47c2ccb2cbada41970b +Author: Jason Tedor +Date: Mon Sep 30 14:05:24 2019 -0400 + + Make Setting#getRaw have private access (#47287) + + The method Setting#getRaw leaks implementation details about settings, + namely that they are backed by strings. We do not want code to rely upon + this, so this commit makes Setting#getRaw private as a first step + towards hiding the implementaton details of settings from the rest of + the codebase. + +commit fd421bd12d0a6795d949a7dba8345fb40defbfcd +Author: James Rodewig +Date: Mon Sep 30 13:54:06 2019 -0400 + + [7.x] [DOCS] Add response body parms to search API docs (#47042) (#47303) + +commit 0c3ee0b15c8c66e1e8ef13642ae8b22afe0c3a7a +Author: Lisa Cawley +Date: Mon Sep 30 10:18:50 2019 -0700 + + [DOCS] Moves Watcher content into Elasticsearch book (#47147) (#47255) + + Co-Authored-By: James Rodewig + +commit 8f1a80a43d48b80e2a393d60e63a0280aa03c91a +Author: Jack Conradson +Date: Mon Sep 30 08:18:34 2019 -0700 + + Move Painless local methods to a dedicated FunctionTable (#46889) + + This moves the way Painless maintains function headers for use + across compilation into its own class - FunctionTable. This + allows us to store a dedicated object for function lookup at + runtime for the def type instead of a loose Map of functions. + +commit 41bc878738b9895a5b263ec5aa62c01202ba4119 +Author: David Turner +Date: Mon Sep 30 16:58:55 2019 +0100 + + Clarify rolling-upgrade docs (#47279) + + Note to upgrade the master-eligible nodes last, and note that + `cluster.initial_master_nodes` should not be set. + +commit 72b63635de5be455e7730e90fd79fb75562fff7c +Author: David Turner +Date: Mon Sep 30 16:58:29 2019 +0100 + + Remove unused pluggable metadata upgraders (#47277) + + Today plugins may provide upgraders for custom metadata and index metadata, but + these upgraders are bypassed during a rolling restart. Fortunately this + extension mechanism is unused by all known plugins. This commit removes these + extension points. + + Relates #47297 + +commit 024d1f2ab91739a2431f4df1afe4cddb57f2768d +Author: James Rodewig +Date: Mon Sep 30 11:38:46 2019 -0400 + + [DOCS] Reformat delete pipeline API (#47172) (#47294) + +commit 312e32a3d72774912bb618f99640a47ce5b4d784 +Author: James Rodewig +Date: Mon Sep 30 11:25:13 2019 -0400 + + [DOCS] Correct snippet in query string syntax + +commit ae85a0e29aa88e273b2b8f340088212e2992fdd6 +Author: Andrew Naguib +Date: Mon Sep 30 17:10:18 2019 +0200 + + [DOCS] Note double backslashes (`\\`) are required to escape JSON chars (#46863) + +commit 24b3703005959634648c761ea9a3c693a434d767 +Author: David Roberts +Date: Mon Sep 30 14:55:50 2019 +0100 + + [TEST] Only wait for 6.6 prerequisites if BWC version is 6.6 or higher (#47289) + + With this change the test setup for ML config upgrade + tests only waits for v6.6+ ML index templates to be + installed if the old cluster is running version 6.6.0 + or higher. + + Previously it was always waiting, but timing out without + failing the test if the templates were not installed + within 10 seconds, effectively just adding a pointless + 10 second sleep to BWC tests against versions earlier + than 6.6.0. This problem was exposed by #47112. + + Fixes #47286 + +commit 052c523d419d8a605332ac1ec4fa338f4bb34f78 +Author: Gaurav614 +Date: Mon Sep 30 18:55:21 2019 +0530 + + Fail allocation of new primaries in empty cluster (#43284) + + Today if you create an index in a cluster without any data nodes then it will + report yellow health because it never attempts to assign any shards if there + are no data nodes, so the new shards remain at `AllocationStatus.NO_ATTEMPT`. + This commit moves the new primaries to `AllocationStatus.DECIDERS_NO` in this + situation, causing the cluster health to move to red. + + Fixes #41073 + +commit 87156ad93badeaa9af0ba90252eef7ef2b561ce2 +Author: emasab +Date: Mon Sep 30 14:55:56 2019 +0200 + + SQL: Fix issue with duplicate columns in SELECT (#42122) + + Previously, if a column (field, scalar, alias) appeared more than once in the + SELECT list, the value was returned only once (1st appearance) in each row. + + Fixes: #41811 + + (cherry picked from commit 097ea36581a751605fc4f2088319d954ce35b5d1) + +commit 5990532cb73c62c317e5995410bb2264872878fa +Author: James Rodewig +Date: Mon Sep 30 08:42:52 2019 -0400 + + [DOCS] Reformat flush API docs (#46875) (#47230) + +commit fe937ea4b8c702c105ab6a1b6afa598dac62f1da +Author: Martijn van Groningen +Date: Mon Sep 30 14:36:53 2019 +0200 + + Add config namespace in get policy api response (#47162) + + Currently the policy config is placed directly in the json object + of the toplevel `policies` array field. For example: + + ``` + { + "policies": [ + { + "match": { + "name" : "my-policy", + "indices" : ["users"], + "match_field" : "email", + "enrich_fields" : [ + "first_name", + "last_name", + "city", + "zip", + "state" + ] + } + } + ] + } + ``` + + This change adds a `config` field in each policy json object: + + ``` + { + "policies": [ + { + "config": { + "match": { + "name" : "my-policy", + "indices" : ["users"], + "match_field" : "email", + "enrich_fields" : [ + "first_name", + "last_name", + "city", + "zip", + "state" + ] + } + } + } + ] + } + ``` + + This allows us in the future to add other information about policies + in the get policy api response. + + The UI will consume this API to build an overview of all policies. + The UI may in the future include additional information about a policy + and the plan is to include that in the get policy api, so that this + information can be gathered in a single api call. + + An example of the information that is likely to be added is: + * Last policy execution time + * The status of a policy (executing, executed, unexecuted) + * Information about the last failure if exists + +commit 0807d409bf99a26fbce6f6d9175c614a75cd4ddc +Author: David Roberts +Date: Thu Sep 26 15:24:40 2019 +0100 + + [ML] Reinstate ML daily maintenance actions (#47103) + + A refactoring in 6.6 meant that the ML daily + maintenance actions have not been run at all + since then. This change installs the local + master listener that schedules the ML daily + maintenance, and also defends against some + subtle race conditions that could occur in the + future if a node flipped very quickly between + master and non-master. + + Fixes #47003 + +commit 467596871a1cd84e0af13ece520e0e247031df35 +Author: Yannick Welsch +Date: Mon Sep 30 13:42:46 2019 +0200 + + Omit writing index metadata for non-replicated closed indices on data-only node (#47285) + + Fixes a bug related to how "closed replicated indices" (introduced in 7.2) interact with the index + metadata storage mechanism, which has special handling for closed indices (but incorrectly + handles replicated closed indices). On non-master-eligible data nodes, it's possible for the + node's manifest file (which tracks the relevant metadata state that the node should persist) to + become out of sync with what's actually stored on disk, leading to an inconsistency that is then + detected at startup, refusing for the node to start up. + + Closes #47276 + +commit d9a7bcef21c743d6b6476805294c951608e74dcc +Author: Przemyslaw Gomulka +Date: Mon Sep 30 13:54:52 2019 +0200 + + Support optional parsers in any order with DateMathParser Backport(46654) (#47217) + + Currently DateMathParser with roundUp = true is relying on the DateFormatter build with + combined optional sub parsers with defaulted fields (depending on the formatter). + That means that for yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSS + Java.time implementation expects optional parsers in order from most specific to + least specific (reverse in the example above). + It is causing a problem because the first parsing succeeds but does not consume the full input. + The second parser should be used. + We can work around this with keeping a list of RoundUpParsers and iterate over them choosing + the one that parsed full input. The same approach we used for regular (non date math) in + relates #40100 + The jdk is not considering this to be a bug https://bugs.openjdk.java.net/browse/JDK-8188771 + + Those below will expect this change first + relates #46242 + relates #45284 + backport #46654 + +commit 7eed3e558530caecc274dec68e37cd59d07d700c +Author: Jason Tedor +Date: Mon Sep 30 06:33:39 2019 -0400 + + Remove use of Setting#getRaw in deprecation tests (#47262) + + This commit removes a use of Setting#getRaw from the deprecation header + tests. The use of Setting#getRaw is not needed here, the x-content + infrastructure will take care of emitting the appropriate values here, + and so the caller does not need to convert these to string + representations of the settings values. + +commit 2cba323b4e08bd8dfa357bacc5f53e16df510c78 +Author: Jason Tedor +Date: Mon Sep 30 06:33:01 2019 -0400 + + Remove use of get raw in token/API key settings (#47260) + + These settings were using get raw to fallback to whether or not SSL is + enabled. Yet, we have a formal mechanism for falling back to a + setting. This commit cuts over to that formal mechanism. + +commit a1d3711b5200d3bf4f374fa2b40ea60592dbfc51 +Author: David Roberts +Date: Mon Sep 30 11:23:04 2019 +0100 + + [TEST] Mute MlConfigIndexMappingsFullClusterRestartIT.testMlConfigIndexMappingsAfterMigratio + + Due to https://github.com/elastic/elasticsearch/issues/47286 + +commit 9dc90e41fc4339c58a41009c54bb8af459d10ff7 +Author: Yannick Welsch +Date: Mon Sep 30 11:58:11 2019 +0200 + + Remove "force" version type (#47228) + + It's been deprecated long ago and can be removed. + + Relates to #20377 + + Closes #19769 + +commit b270f70be16ed03cf70c645a373a4478cb6c4019 +Author: Ryan Ernst +Date: Mon Sep 30 02:04:57 2019 -0700 + + Wait for elasticsearch exit in packaging tests (#47242) + + The archives stopElasticsearch utility method sends SIGTERM to the + elasticsearch process, but does not wait for it to exit. That can cause + subsequent tests to sometimes file. This commit adds wait logic to both + linux and windows for the stopElasticsearch method. + + closes #44501 + +commit 33ebf5f5dc800f19334a09adb295354a0f8a3682 +Author: Ryan Ernst +Date: Mon Sep 30 02:05:29 2019 -0700 + + Fix archive tests to not create elasticsearch home (#47239) + + The test for java home with special characters on linux would create a + temporary java home under /home/elasticsearch. But our packaging + assertions expect that to not exist. Unfortunately this would fail much + later when the checks were actually done in bats tests. This commit + fixes the linux test to match the behavior of windows, which links the + entire java directory, and now does it into a /tmp dir. + + closes #45903 + +commit 3648481eb2017bcef06f782234c3167854f64c43 +Author: Alpar Torok +Date: Mon Sep 30 10:23:04 2019 +0300 + + Replace heavy weight build-tools integ tests with a simple unit test. (#44056) + + Remove heavy build-tool integ test. + Add a unit test for the plugin builder plugin. + + Closes #41256 #41256 + +commit 1c7dc3a5bf3a771e8ec79342bc17f603ca634b4c +Author: Rory Hunter +Date: Thu Sep 26 10:11:30 2019 +0100 + + Format Java source files automatically (#46745) + + This commit adds a Java source formatter and checker into the build process. + This is not yet enabled for any sub-projects - to format and check a + sub-project, add its Gradle path into `build.gradle` and run: + + ./gradlew spotlessApply + + to format, and: + + ./gradlew spotlessJavaCheck + # or: + ./gradlew precommit + + to verify formatting. + +commit bb3e9cb908c45b9365f15ecb4e3a2ff8d5bb0aaf +Author: Martijn van Groningen +Date: Mon Sep 30 08:41:34 2019 +0200 + + fixed checkstyle violation + +commit d229e15b8d2b3e0e427fc59b3502cbe53ae64e4b +Author: Alpar Torok +Date: Mon Sep 30 09:34:11 2019 +0300 + + Add workaround for building docker on debian 8 (#47106) + + Looks like there's a workaround with aufs used in debian 8. + Adding `tsflags=nodocs` works around this issue and results in smaller + image files also. + + Closes #47097 and elastic/infra#14780 + +commit 66f72bcdbc0dacd588e8999bb62dad6cf8dfdebc +Merge: 1c3d5b77b55 2be351c5d09 +Author: Martijn van Groningen +Date: Mon Sep 30 08:12:28 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 1c3d5b77b55337e10840f41f10889e9cb44f6046 +Author: Martijn van Groningen +Date: Mon Sep 30 07:55:23 2019 +0200 + + give monitoring more time + +commit 36215bd33e6e451dfb3575dfc93a64868bdc3401 +Author: Martijn van Groningen +Date: Mon Sep 30 08:04:18 2019 +0200 + + fixed docs issue + +commit 2be351c5d09e0cdce9364b974a235a911ba6af5e +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Sep 30 14:14:52 2019 +1000 + + Use 'should' clause instead of 'filter' when querying native privileges (#47019) (#47271) + + When we added support for wildcard application names, we started to build + the prefix query along with the term query but we used 'filter' clause + instead of 'should', so this would not fetch the correct application + privilege descriptor thereby failing the _has_privilege checks. + This commit changes the clause to use should and with minimum_should_match + as 1. + +commit cec2ff5ef49a487dcd8460634bba8c2f925010d1 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Sep 26 17:56:48 2019 +1000 + + Enhance docs for create api keys created when role descriptor not specified (#46897) + + This commit adds the documentation to point the user that when one + creates API keys with no role descriptor specified then that API + key will have a point in time snapshot of user permissions. + + Closes#46876 + +commit 53a4d2176fa22de2554ab9db2d0cb3c9cebe81e4 +Author: Rory Hunter +Date: Sun Sep 29 12:21:46 2019 +0100 + + Convert most awaitBusy calls to assertBusy (#45794) (#47112) + + Backport of #45794 to 7.x. Convert most `awaitBusy` calls to + `assertBusy`, and use asserts where possible. Follows on from #28548 by + @liketic. + + There were a small number of places where it didn't make sense to me to + call `assertBusy`, so I kept the existing calls but renamed the method to + `waitUntil`. This was partly to better reflect its usage, and partly so + that anyone trying to add a new call to awaitBusy wouldn't be able to find + it. + + I also didn't change the usage in `TransportStopRollupAction` as the + comments state that the local awaitBusy method is a temporary + copy-and-paste. + + Other changes: + + * Rework `waitForDocs` to scale its timeout. Instead of calling + `assertBusy` in a loop, work out a reasonable overall timeout and await + just once. + * Some tests failed after switching to `assertBusy` and had to be fixed. + * Correct the expect templates in AbstractUpgradeTestCase. The ES + Security team confirmed that they don't use templates any more, so + remove this from the expected templates. Also rewrite how the setup + code checks for templates, in order to give more information. + * Remove an expected ML template from XPackRestTestConstants The ML team + advised that the ML tests shouldn't be waiting for any + `.ml-notifications*` templates, since such checks should happen in the + production code instead. + * Also rework the template checking code in `XPackRestTestHelper` to give + more helpful failure messages. + * Fix issue in `DataFrameSurvivesUpgradeIT` when upgrading from < 7.4 + +commit 98989f7b37cf0c39fd7fa8471f6c9d5e4e971254 +Author: Jason Tedor +Date: Sat Sep 28 08:06:11 2019 -0400 + + Use fallback settings in throttling decider (#47261) + + This commit replaces some uses of Setting#getRaw in the throttling + allocation decider settings. Instead, these settings should be using + fallback settings. + +commit bd603b0a7ba4aecf9a7c8f4bca849063eaf75735 +Author: Jason Tedor +Date: Sat Sep 28 08:04:36 2019 -0400 + + Remove dead leniency in allow rebalance setting use (#47259) + + This commit removes some leniency that exists in getting the allow + rebalance setting. Fortunately, that leniency is dead code, this can + never happen. The reason this can never happen is because the settings + infrastructure will not allow setting an invalid value for this + setting. If you try to set this in the elasticsearch.yml, then the node + will fail to start, since parsing the setting will fail. If you try to + set this via an update settings API call, then parsing the setting will + fail and the settings update will be rejected. Therefore, this leniency + can never be activated, so we remove it. + + This commit is the first of a few in an attempt to remove the public + uses of Setting#getRaw. + +commit ab871add36b30fcfd4f7a0377417cdb85e507445 +Author: lcawl +Date: Fri Sep 27 17:10:03 2019 -0700 + + [DOCS] Skip code snippet testing in monitoring example + +commit 62baa85632f4eee76ef6efbad2a878b648b6b888 +Author: Lisa Cawley +Date: Fri Sep 27 14:58:10 2019 -0700 + + [DOCS] Adds production monitoring page (#47184) + +commit b98c57ab2197023ee29f23330da6e13bd9a0a5b0 +Author: James Rodewig +Date: Fri Sep 27 16:31:10 2019 -0400 + + [DOCS] Document missing query parms for cat recovery API (#47181) (#47235) + + Documents the following query parameters for + the /_cat/recovery API: + + * `active_only` + * `detailed` + * `index` + +commit e11c56760d4a8a52049427d100dfc04e1c2b0fab +Author: Tim Brooks +Date: Fri Sep 27 13:52:56 2019 -0600 + + Fix bind failure logging for mock transport (#47150) + + Currently the MockNioTransport uses a custom exception handler for + server channel exceptions. This means that bind failures are logged at + the warn level. This commit modifies the transport to use the common + TcpTransport exception handler which will log exceptions at the correct + level. + +commit d09965a6dd3a9c29f8d62715dbeb4b0667f74ac5 +Author: Jack Conradson +Date: Fri Sep 27 10:55:56 2019 -0700 + + Add ClassWriter to Painless writing pass (#47140) + + This the first part of a series to allow nodes to write all of their appropriate + pieces to the class. Currently, nodes must add their bindings, constants, and + functions to main SClass node for delayed writing. This instead adds a + Painless version of ClassWriter to the write pass. The Painless ClassWriter + contains an appropriate ClassVisitor that can be accessed in any node + during the process along with access to the clinit method, and finally a + shortcut for creating new MethodWriter. The next step will be removing the + delayed writing in SClass, and instead, delegate all writing responsibilities to + the nodes. + +commit 444b47ce886ea41b0ab7a7c89f26c9941e6c1dc9 +Author: Nhat Nguyen +Date: Fri Sep 27 11:13:33 2019 -0400 + + Relax maxSeqNoOfUpdates assertion in FollowingEngine (#47188) + + We disable MSU optimization if the local checkpoint is smaller than + max_seq_no_of_updates. Hence, we need to relax the MSU assertion in + FollowingEngine for that scenario. Suppose the leader has three + operations: index-0, delete-1, and index-2 for the same doc Id. MSU on + the leader is 1 as index-2 is an append. If the follower applies index-0 + then index-2, then the assertion is violated. + + Closes #47137 + +commit e01465eb884fc893509e88659ae3c916eb2385b2 +Author: James Rodewig +Date: Fri Sep 27 13:04:14 2019 -0400 + + [DOCS] Correct typo in ICU Analysis plugin description (#47175) (#47219) + +commit b15930527448cdef242f5aa3b7758487ecf18162 +Author: James Rodewig +Date: Fri Sep 27 11:05:55 2019 -0400 + + [DOCS] Add redirect for SLM API docs (#46838) (#46865) + +commit 76b66634e949c6c5e85db163a2a1d18894bfd04f +Author: Martijn van Groningen +Date: Fri Sep 27 15:21:45 2019 +0200 + + put provided argument on the previous line just like in master branch, + that way this doesn't show in the final pr. + +commit e943e279541c8b9fd628a6b0b6535bdb126d794b +Author: David Roberts +Date: Wed Sep 25 12:04:48 2019 +0100 + + Spawn controller processes from a different directory on macOS (#47013) + + This is the Java side of https://github.com/elastic/ml-cpp/pull/593 + with a fallback so that ml-cpp bundles with either the + new or old directory structure work for the time being. + A few days after merging the C++ changes a followup to + this change will be made that removes the fallback. + +commit 7ffe2e7e63c4ce072711e6c57281ccb6d85d05b0 +Merge: 8a4eefdd839 3fbd58d156c +Author: Martijn van Groningen +Date: Fri Sep 27 14:42:11 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 01623f9f1c39d23665865235ebbaf0f805c4f748 +Author: Marios Trivyzas +Date: Fri Sep 27 15:38:03 2019 +0300 + + SQL: Add alias DATETRUNC to DATE_TRUNC function (#47173) + + To be on the safe side in terms of use cases also add the alias + DATETRUNC to the DATE_TRUNC function. + + Follows: #46473 + + (cherry picked from commit 9ac223cb1fc66486f86e218fa785a32b61e9bacc) + +commit 4c909438dd8ba63198c8fc38b6f725cb4b69545c +Author: Andrei Dan +Date: Fri Sep 27 13:16:45 2019 +0100 + + Fix OriginationDate parsing tests. (#47170) (#47200) + + Drop the usage of `SimpleDateFormat` and use the `DateFormatter` instead + + (cherry picked from commit 7cf509a7a11ecf6c40c44c18e8f03b8e81fcd1c2) + Signed-off-by: Andrei Dan + +commit 6fd3b4723f94f3b5e7a699f208252e99eebd25da +Author: Yannick Welsch +Date: Fri Sep 27 13:57:37 2019 +0200 + + Remove write lock for Translog.getGeneration (#47036) + + No need for the write lock, and currentFileGeneration is already protected by the read lock. + Also removes the unused method "isCurrent". + +commit 3fbd58d156c420bfbf3e1070b314ac0f465e6c14 +Author: Przemysław Witek +Date: Fri Sep 27 13:01:51 2019 +0200 + + [7.x] Allow evaluation to consist of multiple steps. (#46653) (#47194) + +commit a1e2e208ce5cad4829e7ac160e95d995f31c5870 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Sep 27 12:05:41 2019 +0200 + + Mute Snapshot/Restore with repository-azure (#47204) + + Relates #47201 + +commit 0ab7132c4741ab8d275cb87cd684744265a6e642 +Author: István Zoltán Szabó +Date: Fri Sep 27 10:34:30 2019 +0200 + + [DOCS] Reformats Profile API (#47168) + + * [DOCS] Reformats Profile API. + + * [DOCS] Fixes failing docs test. + +commit 2cd6bed9c8a09831ab2afdd0abb784348c9b90b8 +Author: Colin Goodheart-Smithe +Date: Fri Sep 27 09:18:59 2019 +0100 + + Updates 7.4.0 release notes + +commit 42ae76ab7ca711fbb6adc2c4a9a01f37aa21fa2e +Author: Tanguy Leroux +Date: Fri Sep 27 09:30:00 2019 +0200 + + Injected response errors in Azure repository tests should have a body (#47176) + + The Azure SDK client expects server errors to have a body, + something that looks like: + + + + string-value + string-value + + + I've forgot to add such errors in Azure tests and that triggers + some NPE in the client like the one reported in #47120. + + Closes #47120 + +commit 813b130e08ce0a6f08368e4ec831981f8dbb3ecf +Author: Alpar Torok +Date: Fri Sep 27 10:34:51 2019 +0300 + + Exclude the demo folder form the JDK (#47161) + + The folder contains jars with source code that fail the lintian test on + debian (based) distributions. + +commit 944421627d34f88e967ecd91520887a4923b0e3e +Author: Alpar Torok +Date: Fri Sep 27 09:03:15 2019 +0300 + + Work around error building deb on Windows (#47011) + + Relates to #47007 . the `gradle-ospackage-plugin` plugin doesn't + properly support symlink on windows. + + This PR changes the way we configure tasks to prevent building these + packages as part of a windows check. + +commit 9c60e15ebd02d4b6bc44881f26e0eb20ef0c0d98 +Author: Pius +Date: Thu Sep 26 13:14:42 2019 -0700 + + [DOCS] Add 'remote clusters' requirement link to CCR docs (#47185) + +commit 4714f106502a1b1e0dbff084d83ebdc7bbb661a9 +Author: Nicole Albee <2642763+a03nikki@users.noreply.github.com> +Date: Thu Sep 26 14:48:54 2019 -0500 + + [DOCS] Remove coming tag from 7.2.0 release highlights (#47183) + +commit b29a2cb360a07b768acbb704e885096f77d65bc4 +Author: Costin Leau +Date: Thu Sep 26 19:47:55 2019 +0300 + + SQL: Check case where the pivot limit is reached (#47121) + + In some cases, the fetch size affects the way the groups are returned + causing the last page to go beyond the limit. Add dedicated check to + prevent extra data from being returned. + + Fix #47002 + + (cherry picked from commit f4c29646f097bbd29855300342823ef4cef61c05) + +commit 9b4f37747472894538640381e2369dfa97598712 +Author: Jack Conradson +Date: Thu Sep 26 10:31:25 2019 -0700 + + Change Painless function node to use a block instead of raw statements (#46884) + + This change improves the node structure of SFunction. SFunction now uses + an SBlock instead of a List of AStatments reducing code duplication and + gives a future target for symbol table scoping. + +commit ae202fda214c145c75990a85baf586be05a79e10 +Author: Igor Motov +Date: Wed Sep 25 13:43:05 2019 -0400 + + SQL: Add support for shape type (#46464) + + Enables support for Cartesian geometries shape type. We still need to + decide how to handle the distance function since it is currently using + the haversine distance formula and returns results in meters, which + doesn't make any sense for Cartesian geometries. + + Closes #46412 + Relates to #43644 + +commit 0c575dc1e8d8dcff28548869f39241b7c5349507 +Author: James Rodewig +Date: Thu Sep 26 08:56:24 2019 -0400 + + [DOCS] Correct link to `index.store.preload` setting (#47145) + +commit 3b626c2d5649fc66431a61e49fc617f15c01b421 +Author: James Rodewig +Date: Thu Sep 26 08:51:12 2019 -0400 + + [DOCS] Reformat get pipeline API (#47131) (#47163) + +commit 77cc6d5bad06ffb53b0422806c4019cd6ff615c2 +Author: David Roberts +Date: Thu Sep 26 13:19:10 2019 +0100 + + [TEST] Work around _cat/indices bug with security enabled (#47160) + + When the ML native multi-node tests use _cat/indices/_all + and the request goes to a non-master node, _all is + translated to a list of concrete indices by the authz layer + on the coordinating node before the request is forwarded + to the master node. Then it is possible for the master + node to return an index_not_found_exception if one of + the concrete indices that was expanded on the + coordinating node has been deleted in the meantime. + (#47159 has been opened to track the underlying problem.) + + It has been observed that the index that gets deleted when + the problem affects the ML native multi-node tests is + always the ML notifications index. The tests that fail are + only interested in the presence or absense of ML results + indices. Therefore the workaround is to only _cat indices + that match the ML results index pattern. + + Fixes #45652 + +commit 0765bd4bf7d5d1f4a1fe5497d72069fe4b848f68 +Author: Dimitris Athanasiou +Date: Thu Sep 26 15:26:06 2019 +0300 + + [7.x][ML] Ensure data frame analytics task is only marked completed once (#47119) (#47157) + + Closes #46907 + +commit 73a09b34b82591d779bcf24094cf07960fb1c44d +Author: Jim Ferenczi +Date: Thu Sep 26 14:21:23 2019 +0200 + + Replace SearchContextException with SearchException (#47046) + + This commit removes the SearchContextException in favor of a simpler + SearchException that doesn't leak the SearchContext. + + Relates #46523 + +commit 95e2ca741eb6d73e1f36bb17e1a44afdef61ca58 +Author: Tanguy Leroux +Date: Thu Sep 26 12:49:21 2019 +0200 + + Remove unused private methods and fields (#47154) + + This commit removes a bunch of unused private fields and unused + private methods from the code base. + + Backport of (#47115) + +commit 8a4eefdd839d6592294949d782173e578b90ab9e +Author: Martijn van Groningen +Date: Thu Sep 26 11:00:33 2019 +0200 + + Expose enrich stats api to monitoring. (#46708) + + This change also slightly modifies the stats response, + so that is can easier consumer by monitoring and other + users. (coordinators stats are now in a list instead of + a map and has an additional field for the node id) + + Relates to #32789 + +commit 9a64b7a888e4b95845d4611814a12297dc7d5319 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Sep 26 17:57:36 2019 +1000 + + [Backport] Validate `query` field when creating roles (#46275) (#47094) + + In the current implementation, the validation of the role query + occurs at runtime when the query is being executed. + + This commit adds validation for the role query when creating a role + but not for the template query as we do not have the runtime + information required for evaluating the template query (eg. authenticated user's + information). This is similar to the scripts that we + store but do not evaluate or parse if they are valid queries or not. + + For validation, the query is evaluated (if not a template), parsed to build the + QueryBuilder and verify if the query type is allowed. + + Closes #34252 + +commit 97d977f381a9870ebda6ba26a9d453174983d5d1 +Author: jimczi +Date: Thu Sep 26 09:56:24 2019 +0200 + + #47046 Fix serialization version check after backport + +commit 04972baffa96c19c1601cfb245a8458195db1693 +Author: Jim Ferenczi +Date: Thu Sep 26 09:20:53 2019 +0200 + + Merge ShardSearchTransportRequest and ShardSearchLocalRequest (#46996) (#47081) + + This change merges the `ShardSearchTransportRequest` and `ShardSearchLocalRequest` + into a single `ShardSearchRequest` that can be used to create a SearchContext. + + Relates #46523 + +commit 429f23ea2f25f05a5c1a4ab54f0a4c45a7d723ff +Author: Martijn van Groningen +Date: Thu Sep 26 08:55:28 2019 +0200 + + Allow ingest processors to execute in a non blocking manner. (#47122) + + Backport of #46241 + + This PR changes the ingest executing to be non blocking + by adding an additional method to the Processor interface + that accepts a BiConsumer as handler and changing + IngestService#executeBulkRequest(...) to ingest document + in a non blocking fashion iff a processor executes + in a non blocking fashion. + + This is the second PR that merges changes made to server module from + the enrich branch (see #32789) into the master branch. + + The plan is to merge changes made to the server module separately from + the pr that will merge enrich into master, so that these changes can + be reviewed in isolation. + + This change originates from the enrich branch and was introduced there + in #43361. + +commit 45c77830181dacfe0e770238a4163dd19c39d1a7 +Author: David Turner +Date: Thu Sep 26 07:40:54 2019 +0100 + + Warn on slow metadata persistence (#47130) + + Today if metadata persistence is excessively slow on a master-ineligible node + then the `ClusterApplierService` emits a warning indicating that the + `GatewayMetaState` applier was slow, but gives no further details. If it is + excessively slow on a master-eligible node then we do not see any warning at + all, although we might see other consequences such as a lagging node or a + master failure. + + With this commit we emit a warning if metadata persistence takes longer than a + configurable threshold, which defaults to `10s`. We also emit statistics that + record how much index metadata was persisted and how much was skipped since + this can help distinguish cases where IO was slow from cases where there are + simply too many indices involved. + + Backport of #47005. + +commit fcddaa90dea6baecce4889014896934218ebf1b7 +Author: Benjamin Trent +Date: Wed Sep 25 19:11:15 2019 -0400 + + [7.x] [ML][Inference] adding tree model (#47044) (#47141) + + * [ML][Inference] adding tree model (#47044) + + * [ML][Inference] adding tree model + + * renaming features for updated schema + + * fixing 7.x compilation + +commit 7ac647c365c883e8f1a5b952893e68b8ba3b8412 +Author: Gordon Brown +Date: Wed Sep 25 16:15:10 2019 -0600 + + Add support for POST requests to SLM Execute API (#47061) + + This commit adds support for POST requests to the SLM `_execute` API, + because POST is a more appropriate HTTP verb for this action as it is + not idempotent. The docs are also changed to favor POST over PUT, + although PUT is not removed or officially deprecated. + +commit 4f47e1f1695afedcef6f5ade99c3fde717a86c20 +Author: Tim Brooks +Date: Wed Sep 25 15:58:18 2019 -0600 + + Extract proxy connection logic to specialized class (#47138) + + Currently the logic to check if a connection to a remote discovery node + exists and otherwise create a proxy connection is mixed with the + collect nodes, cluster connection lifecycle, and other + RemoteClusterConnection logic. This commit introduces a specialized + RemoteConnectionManager class which handles the open connections. + Additionally, it reworks the "round-robin" proxy logic to create the list + of potential connections at connection open/close time, opposed to each + time a connection is requested. + +commit 7c5a088aa58f25bf56e986c6a930611852c8ac3c +Author: Nhat Nguyen +Date: Wed Sep 25 16:52:19 2019 -0400 + + Increase ensureGreen timeout for testReplicaCorruption (#47136) + + We can have a large number of shard copies in this test. For example, + the two recent failures have 24 and 27 copies respectively and all + replicas have to copy segment files as their stores are corrupted. Our + CI needs more than 30 seconds to start all these copies. + + Note that in two recent failures, the cluster was green just after the + cluster health timed out. + + Closes #41899 + +commit 27520cac3ba8360d5d40a44bcd0a5b4074f7ef53 +Author: Andrei Dan +Date: Wed Sep 25 21:44:16 2019 +0100 + + ILM: parse origination date from index name (#46755) (#47124) + + * ILM: parse origination date from index name (#46755) + + Introduce the `index.lifecycle.parse_origination_date` setting that + indicates if the origination date should be parsed from the index name. + If set to true an index which doesn't match the expected format (namely + `indexName-{dateFormat}-optional_digits` will fail before being created. + The origination date will be parsed when initialising a lifecycle for an + index and it will be set as the `index.lifecycle.origination_date` for + that index. + + A user set value for `index.lifecycle.origination_date` will always + override a possible parsable date from the index name. + + (cherry picked from commit c363d27f0210733dad0c307d54fa224a92ddb569) + Signed-off-by: Andrei Dan + + * Drop usage of Map.of to be java 8 compliant + +commit a267df30fa4d427d84e426679f91735556e67b98 +Author: Lee Hinman +Date: Wed Sep 25 14:24:54 2019 -0600 + + Wait for snapshot completion in SLM snapshot invocation (#47051) + + * Wait for snapshot completion in SLM snapshot invocation + + This changes the snapshots internally invoked by SLM to wait for + completion. This allows us to capture more snapshotting failure + scenarios. + + For example, previously a snapshot would be created and then registered + as a "success", however, the snapshot may have been aborted, or it may + have had a subset of its shards fail. These cases are now handled by + inspecting the response to the `CreateSnapshotRequest` and ensuring that + there are no failures. If any failures are present, the history store + now stores the action as a failure instead of a success. + + Relates to #38461 and #43663 + +commit 287d96d1a11ea3dcb324033f85ed580d40b8f09d +Author: Lisa Cawley +Date: Wed Sep 25 13:02:28 2019 -0700 + + [DOCS] Fix links to transform pages (#47134) + +commit d0495b8f933195bd16ff621e141449cd20a341f5 +Author: Mark Vieira +Date: Wed Sep 25 11:19:39 2019 -0700 + + Track enabled test task candidate class files as task input (#47054) + +commit a46eef9634ada3bbe111aa13b24ecd05fb0c8e9f +Author: Gordon Brown +Date: Wed Sep 25 11:32:08 2019 -0600 + + Change SLM stats format (#46991) + + Using arrays of objects with embedded IDs is preferred for new APIs over + using entity IDs as JSON keys. This commit changes the SLM stats API to + use the preferred format. + +commit 35f056661f2357f10b8fa42e53f6c7c7d392b7be +Author: Colin Goodheart-Smithe +Date: Wed Sep 25 08:53:33 2019 +0100 + + Updates 7.4.0 release notes + +commit 9ddc99de230809ea0e3c1855a8c1059405d01d98 +Author: James Rodewig +Date: Wed Sep 25 12:36:23 2019 -0400 + + [DOCS] Reformat clone index API docs (#46762) + +commit 365aa30b7f7a271ab7e0259faf334a8e3e3dfb8e +Author: James Rodewig +Date: Wed Sep 25 12:35:48 2019 -0400 + + [DOCS] Remove support for `// CONSOLE` magic comment (#46936) (#47110) + + #46180 added support for the `[source,console]` + language for snippets which should be tested. + + This removes support for the `// CONSOLE` magic comment, + which serve a similar purpose. + + Snippets that include the `// CONSOLE` magic comment will return + an exception. + +commit d5f396fe5571c34c11f625a2be1e471125a155d9 +Author: Lisa Cawley +Date: Wed Sep 25 08:11:37 2019 -0700 + + [DOCS] Adds transforms to Elasticsearch book (#46846) (#47055) + +commit 93fcd23da8aca351a03317d52f1d37b5deb3d941 +Author: Armin Braun +Date: Wed Sep 25 15:55:33 2019 +0200 + + Fail Snapshot on Corrupted Metadata Blob (#47009) (#47096) + + We should not be quietly ignoring a corrupted shard-level index-N + blob. Simply creating a new empty shard-level index-N and moving + on means that all snapshots of that shard show `SUCESS` as their + state at the repository root but are in fact broken. + This change at least makes it visible to the user that they can't + snapshot the given shard any more and forces the user to move on + to a new repository since the current one is broken and will not + allow snapshotting the inconsistent shard again. + + Also, this change stops the delete action for shards with broken + index-N blobs instead of simply deleting all blobs in the path + containing the broken index-N. This prevents a temporarily + broken/missing index-N blob from corrupting all snapshots of + that shard. + +commit 58d2bf7e09f1e43d5aebc46958ed0ade2a40d2cb +Author: Ioannis Kakavas +Date: Wed Sep 25 16:05:42 2019 +0300 + + Re-enable BWC tests (#47102) + + Re-enable BWC tests now that #46534 has been backported to 7.x + +commit 74fd21f0b0b694ad18e8a719fb6ea759f86a28ee +Author: István Zoltán Szabó +Date: Wed Sep 25 14:55:09 2019 +0200 + + [DOCS] Reformats ranking evaluation API (#46974) + + * [DOCS] Reformats ranking evaluation API. + Co-Authored-By: James Rodewig + +commit 22575bd7e692c2d79d98ca20f5e647fd4e88c591 +Author: Nhat Nguyen +Date: Wed Sep 25 08:25:14 2019 -0400 + + Remove isRecovering method from Engine (#47039) + + We already prevent flushing in Engine if it's recovering. Hence, we can + remove the protection in IndexShard. + +commit c4a166fc9a6fb467ea3048516dba17bfde2e548e +Author: Armin Braun +Date: Wed Sep 25 14:53:11 2019 +0200 + + Simplify SnapshotResiliencyTests (#46961) (#47108) + + Simplify `SnapshotResiliencyTests` to more closely + match the structure of `AbstractCoordinatorTestCase` and allow for + future drying up between the two classes: + + * Make the test cluster nodes a nested-class in the test cluster itself + * Remove the needless custom network disruption implementation and + simply track disconnected node ids like `AbstractCoordinatorTestCase` + does + +commit 83365e94babfc03d9f88337b4d0e7429539ee175 +Author: István Zoltán Szabó +Date: Wed Sep 25 14:38:47 2019 +0200 + + [DOCS] Reformat suggesters page. (#47010) + +commit 9e17b78fee4c727f29b94b86517a0fb2fed12497 +Author: Yannick Welsch +Date: Wed Sep 25 14:14:10 2019 +0200 + + Mute second test in monitoring/bulk/10_basic + + Relates #30101 + +commit 05fb7be5716949b672db2a5307d5f7b0fbf1d0cf +Author: Benjamin Trent +Date: Wed Sep 25 08:16:24 2019 -0400 + + [7.x] [ML][Inference] Feature pre-processing objects and functions (#46777) (#47040) + + * [ML][Inference] Feature pre-processing objects and functions (#46777) + + To support inference on pre-trained machine learning models, some basic feature encoding will be necessary. I am using a named object serialization approach so new encodings/pre-processing steps could be added in the future. + + This PR lays down the ground work for 3 basic encodings: + + * HotOne + * Target Mean + * Frequency + + More feature encodings or pre-processings could be added in the future: + + * Handling missing columns + * Standardization + * Label encoding + * etc.... + + * fixing compilation for namedxcontent tests + +commit 81cbd3fba4849c108c129134bc6c4243deab9540 +Author: Yannick Welsch +Date: Wed Sep 25 14:01:54 2019 +0200 + + Mute ClusterShardLimitIT.testIndexCreationOverLimitFromTemplate + + Relates #47107 + +commit a4cecc54aba6c7c2c4d99cefea6810cb51b7a4c7 +Author: Yannick Welsch +Date: Wed Sep 25 14:01:09 2019 +0200 + + Mute monitoring/bulk/20_privileges + + Relates #30101 + +commit ac920e8e645cdcc4320f79d7c11614e4da8ffe62 +Author: David Turner +Date: Wed Sep 25 12:31:36 2019 +0100 + + Assert no exceptions during state application (#47090) + + Today we log and swallow exceptions during cluster state application, but such + an exception should not occur. This commit adds assertions of this fact, and + updates the Javadocs to explain it. + + Relates #47038 + +commit eb86d71edd8dd4839c78b0fa3c2a32fd02c2ba8d +Author: Yannick Welsch +Date: Wed Sep 25 12:51:24 2019 +0200 + + Mute MlJobIT.testDeleteJob + + Relates #45652 + +commit 7a5b5af1719fbb10114bc08c90e87c870608db8f +Author: Yannick Welsch +Date: Wed Sep 25 11:50:36 2019 +0200 + + Mute MlJobIT.testDeleteJobAsync + + Relates #45652 + +commit f785c31531acd2661c4df2c47b60b3e7fa16bf4b +Author: Ioannis Kakavas +Date: Wed Sep 25 13:52:05 2019 +0300 + + File based role definition documentation additions (#46304) (#47085) + + This commit clarifies and points out that the Role management UI and + the Role management API cannot be used to manage roles that are + defined in roles.yml and that file based role management is + intended to have a small administrative scope and not handle all + possible RBAC use cases. + +commit 23bceaadf87111047c2df592484cdcf8711a095c +Author: Ioannis Kakavas +Date: Wed Sep 25 13:23:46 2019 +0300 + + Handle RelayState in preparing a SAMLAuthN Request (#46534) (#47092) + + This change allows for the caller of the `saml/prepare` API to pass + a `relay_state` parameter that will then be part of the redirect + URL in the response as the `RelayState` query parameter. + + The SAML IdP is required to reflect back the value of that relay + state when sending a SAML Response. The caller of the APIs can + then, when receiving the SAML Response, read and consume the value + as it see fit. + +commit eef1ba3fadfa3adae2be7280a97f1812a39c1f1a +Author: Martijn van Groningen +Date: Wed Sep 25 10:35:36 2019 +0200 + + Make ingest pipeline resolution logic unit testable (#47026) + + Extracted ingest pipeline resolution logic into a static method + and added unit tests for pipeline resolution logic. + + Followup from #46847 + +commit b234d2cbd778ec9c49d8ec8f7a37753653c8037c +Author: Paweł Krześniak +Date: Wed Sep 25 11:23:52 2019 +0200 + + [Docs] Fix the numbering in the annotated_text example (#47077) + +commit 6f453aa6b2eae90a72014125c3ef178a00f40e77 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Sep 25 18:57:11 2019 +1000 + + Validate index and cluster privilege names when creating a role (#46361) (#47063) + + This commit adds validation so a role cannot be created with + invalid index or cluster privilege name. + + Closes #29703 + +commit 056ac32738fc94ef1787ae2b718402782b81a701 +Author: Yannick Welsch +Date: Wed Sep 25 10:34:54 2019 +0200 + + Mute JdbcCsvSpecIT.testAverageWithOneValueAndLimit + + Relates to #47080 + +commit 48df560593a5f0dcf039a0ea2772c60eccbaa2ae +Author: Daniel Mitterdorfer +Date: Wed Sep 25 10:22:46 2019 +0200 + + Emit log message when parent circuit breaker trips (#47000) (#47073) + + We emit a debug log message whenever a child circuit breaker trips (in + `ChildMemoryCircuitBreaker#circuitBreak(String, long)`) but we never + emit a log message when the parent circuit breaker trips. As this is + more likely to happen with the real memory circuit breaker it is not + possible to detect this in the logs. With this commit we add a log + message on the same log level (debug) when the parent circuit breaker + trips. + +commit 0c187e0a103df6c11efcc395caa5e4f3fa6bea80 +Author: Christoph Büscher +Date: Wed Sep 25 10:15:10 2019 +0200 + + Add migration tool checks for `_field_names` disabling (#46972) + + This change adds a check to the migration tool that warns about the deprecated + `enabled` setting for the `_field_names` field on 7.x indices and issues a + warning for templates containing this setting, which has been removed + with 8.0. + + Relates to #42854, #46681 + +commit db63e78b68c6638d92ce181f7f900d2ac5342326 +Author: Yannick Welsch +Date: Wed Sep 25 09:01:15 2019 +0200 + + Mute DebMetadataTests.test05CheckLintian + + Relates #46903 + +commit b1bf05bb8985cd6afcc03483678abfd981d3eb13 +Author: Tanguy Leroux +Date: Wed Sep 25 09:18:44 2019 +0200 + + Add blob container retries tests for Azure SDK client (#47032) + + Similarly to what has been done for S3 and GCS, this commit + adds unit tests that verify the retry logic of the Azure SDK + client implementation when the remote service returns errors. + + It only tests the retry logic in case of errors and not in + case of timeouts because Azure client timeout options are + not exposed as settings. + +commit 7377ac4637f6c509f17b7e911145830dda1b7046 +Author: Hendrik Muhs +Date: Wed Sep 25 07:38:59 2019 +0200 + + [Transform] Replace transforms with transform, index constants (#47023) + + - replace "transforms" with "transform" for consistency + - use constants for internal index naming wherever possible and document required changes + +commit e974f178b57833c8cf6b02f440abd7a080859783 +Author: Hendrik Muhs +Date: Wed Sep 25 07:38:17 2019 +0200 + + [Transform] rename data frame transform to transform for hlrc client (#46933) + + rename data frame transform to transform for hlrc + +commit 61eef2fd3139791e61889c9aee18493aeec873ed +Author: James Rodewig +Date: Wed Sep 25 02:01:37 2019 -0400 + + [DOCS] Add multi-level nested query example to nested query docs (#46986) + +commit 4faba9cbbfa67d1b15dd8e7f93be6d255b4b801f +Author: Julie Tibshirani +Date: Tue Sep 24 12:38:23 2019 -0700 + + Mention ip fields in the global ordinals docs. (#47045) + + Although they do not support eager_global_ordinals, ip fields use global + ordinals for certain aggregations like 'terms'. + + This commit also corrects a reference to the sampler aggregation. + +commit 41ee8aa6fcdf4713f31b98b866287b28a4caafa7 +Author: Julie Tibshirani +Date: Tue Sep 24 11:40:21 2019 -0700 + + Reject regexp queries on the _index field. (#46945) + + We speculatively added support for `regexp` queries on the `_index` field in + #34089 (this functionality was not actually requested by a user). Supporting + regex logic adds complexity to the `_index` field for not much gain, so we + would like to remove it. + + From an end-to-end test it turns out this functionality never even worked in + the first place because of an error in how regex flags were interpreted! For + this reason, we can remove support for `regexp` on `_index` without a + deprecation period. + + Relates to #46640. + +commit 1893f6adc290b4308b0bd0a3549ae514a4184071 +Author: Lisa Cawley +Date: Tue Sep 24 10:35:06 2019 -0700 + + [DOCS] Merge monitoring sections in Elasticsearch book (#46885) + +commit 71ec0707cfa196d09837b979e708f1c799ba1f2c +Author: Tim Brooks +Date: Tue Sep 24 10:55:49 2019 -0600 + + Remove locking around connection attempts (#46845) + + Currently in the ConnectionManager we lock around the node id. This is + odd because we key connections by the ephemeral id. Upon further + investigation it appears to me that we do not need the locking. Using + the concurrent map, we can ensure that only one connection attempt + completes. There is a very small chance that a new connection attempt + will proceed right as another connection attempt is completing. However, + since the whole process is asynchronous and event oriented + (lightweight), that does not seem to be an issue. + +commit 00c1c0132b13451625d160272f41bf54ff7d00fe +Author: Benjamin Trent +Date: Tue Sep 24 13:03:20 2019 -0400 + + [ML] fix two datafeed flush lockup bugs (#46982) (#47024) + + * [ML] fix two flush lockup bugs + + * Addressing PR comments + + * moving debug logging line so it is only written on success + +commit 9967aff714c22fe1e8149d694802163bdff4e4d3 +Author: James Baiera +Date: Tue Sep 24 12:26:56 2019 -0400 + + Add notice to Enrich index mapping metadata (#45996) + +commit 6720c56bdd267d726a0dafe921a8df4b26990f60 +Author: Tim Brooks +Date: Thu Aug 29 17:38:37 2019 -0600 + + Set netty system properties in BuildPlugin (#45881) + + Currently in production instances of Elasticsearch we set a couple of + system properties by default. We currently do not apply all of these + system properties in tests. This commit applies these properties in the + tests. + +commit ffae769186f3de39e35d4fef53c48a09997c9d2f +Author: David Roberts +Date: Tue Sep 24 17:38:09 2019 +0100 + + [TEST] Rename data-frames to transform in docs test cleanup (#47027) + + The renaming of the tests in #46760 caused the + cleanup between tests to be skipped. + + Backport of #47016 + +commit f02582de4b03a255c16a28fd785494761a993dba +Author: Tim Brooks +Date: Fri Sep 20 11:09:36 2019 -0600 + + Reduce a bind failure to trace logging (#46891) + + Due to recent changes in the nio transport, a failure to bind the server + channel has started to be logged at an error level. This exception leads + to an automatic retry on a different port, so it should only be logged + at a trace level. + +commit 22dade8e1b2b784a415fe0f87a58fda28bb96d59 +Author: David Roberts +Date: Tue Sep 24 17:20:25 2019 +0100 + + [DOCS] Add 7.4 breaking changes for transforms and data frame analytics (#46821) + + The PRs that made these changes are: + + - https://github.com/elastic/elasticsearch/pull/44350 + - https://github.com/elastic/elasticsearch/pull/45276 + - https://github.com/elastic/elasticsearch/pull/45856 + + Co-Authored-By: István Zoltán Szabó + Co-Authored-By: Lisa Cawley + +commit 3a82e0f7f4173d8e58b9e921d34ee50788e173cb +Author: Albert Zaharovits +Date: Tue Sep 24 19:07:55 2019 +0300 + + Do not rewrite aliases on remove-index from aliases requests (#46989) (#47018) + + When we rewrite alias requests, after filtering down to only those that + the user is authorized to see, it can be that there are no aliases + remaining in the request. However, core Elasticsearch interprets this as + _all so the user would see more than they are authorized for. To address + this, we previously rewrote all such requests to have aliases `"*"`, + `"-*"`, which would be interpreted when aliases are resolved as + nome. Yet, this is only needed for get aliases requests and we were + applying it to all alias requests, including remove index requests. If + such a request was sent to a coordinating node that is not the master + node, the request would be rewritten to include `"*"` and `"-*"`, and + then the master would authorize the user for these. If the user had + limited permissions, the request would fail, even if they were + authorized on the index that the remove index action was over. This + commit addresses this by rewriting for get aliases and remove + aliases request types but not for the remove index. + + Co-authored-by: Albert Zaharovits + Co-authored-by: Tim Vernum + +commit 64bf1b56fea7d59c4c217d917cfd0f58261f9c20 +Author: Dimitris Athanasiou +Date: Tue Sep 24 19:04:52 2019 +0300 + + [7.x] SQL: Mute pivot testAverageWithOneValueAndOrder and testSumWithoutSubquery (#47030) (#47033) + + Relates #47002 + +commit 00f2e7f6270d1c8321f0cb234cb47b858cdbe39d +Author: Armin Braun +Date: Tue Sep 24 17:15:11 2019 +0200 + + Update AWS SDK for repository-s3 plugin to support IAM Roles for Service Accounts (#46969) (#47004) + + * Update AWS SDK for repository-s3 and discovery-ec2 plugins + +commit a1af2fe96ae67968f1d1c393cddd3bbfe98809d2 +Author: Jack Conradson +Date: Tue Sep 24 07:33:55 2019 -0700 + + Rename Painless node SSource to SClass (#46984) + + Mechanical renaming of SSource node to SClass to better align with the names of what other nodes generate. + +commit 9135e2f9e391dbb94e4c8495c4fa6f3e9a00bfbf +Author: David Turner +Date: Tue Sep 24 13:39:44 2019 +0100 + + Improve LeaderCheck rejection messages (#46998) + + Today the `LeaderChecker` rejects checks from nodes that are not in the current + cluster with the exception message `leader check from unknown node` which + offers no information about why the node is unknown. In fact the node must have + been in the cluster in the recent past, so it might help guide the user to a + more useful log message if we describe it as a `removed node` instead of an + `unknown node`. This commit changes the exception message like this, and also + tidies up a few other loose ends in the `LeaderChecker`. + +commit 6943a3101fea1a4a438323c6ec98cf9650e2245c +Author: David Turner +Date: Tue Sep 24 12:31:13 2019 +0100 + + Cut PersistedState interface from GatewayMetaState (#46655) + + Today `GatewayMetaState` implements `PersistedState` but it's an error to use + it as a `PersistedState` before it's been started, or if the node is + master-ineligible. It also holds some fields that are meaningless on nodes that + do not persist their states. Finally, it takes responsibility for both loading + the original cluster state and some of the high-level logic for writing the + cluster state back to disk. + + This commit addresses these concerns by introducing a more specific + `PersistedState` implementation for use on master-eligible nodes which is only + instantiated if and when it's appropriate. It also moves the fields and + high-level persistence logic into a new `IncrementalClusterStateWriter` with a + more appropriate lifecycle. + + Follow-up to #46326 and #46532 + Relates #47001 + +commit 98e6bb4d01de21788e36a88e0dc3d67ed5a25a37 +Author: Ioannis Kakavas +Date: Tue Sep 24 12:47:56 2019 +0300 + + Workaround JDK-8213202 in SSLClientAuthTests (#46995) + + This change works around JDK-8213202, which is a bug related to TLSv1.3 + session resumption before JDK 11.0.3 that occurs when there are + multiple concurrent sessions being established. Nodes connecting to + each other will trigger this bug when client authentication is + disabled, which is the case for SSLClientAuthTests. + + Backport of #46680 + +commit 56224068d46f87654b434431c59cc6454f8e7531 +Author: Colin Goodheart-Smithe +Date: Tue Sep 24 09:23:25 2019 +0100 + + Release highlights for 7.4.0 (#46963) + + Co-Authored-By: James Rodewig + +commit 618efcfcf94a5803b4974113a9e572b7bb23011f +Author: maidoo +Date: Tue Sep 24 16:01:07 2019 +0800 + + Add submitDeleteByQueryTask method to RestHighLevelClient (#46833) + + The HLRC has a method for reindex, that allows to trigger an async reindex by running RestHighLevelClient.submitReindexTask and RestHighLevelClient.reindex. The delete by query however only has an RestHighLevelClient.deleteByQuery method (and its async counterpart), but no RestHighLevelClient.submitDeleteByQueryTask. So add RestHighLevelClient.submitDeleteByQueryTask + + Closes #46395 + +commit 6986d7f9680cd20de60ae615c77281a9f955611f +Author: Tanguy Leroux +Date: Tue Sep 24 08:57:39 2019 +0200 + + Add blob container retries tests for Google Cloud Storage (#46968) + + Similarly to what has been done for S3 in #45383, this commit + adds unit tests that verify the behavior of the SDK client and + blob container implementation for Google Storage when the + remote service returns errors. + + The main purpose was to add an extra test to the specific retry + logic for 410-Gone errors added in #45963. + + Relates #45963 + +commit 5ca37db60cc98916b07b0bef98b3c5a8c5420cc7 +Author: Lee Hinman +Date: Mon Sep 23 17:06:27 2019 -0600 + + Mute SLMSnapshotBlockingIntegTests.testRetentionWhileSnapshotInProgress + + Relates to #46508 + +commit a349b222738e49c710d703e72f51a899e470bdd7 +Author: James Baiera +Date: Fri Sep 20 11:39:09 2019 -0400 + + Add the cluster version to enrich policies (#45021) + + Adds the Elasticsearch version as a field on the EnrichPolicy object + +commit 9124c94a6c75b856fbd2c31a5b6bdb94b5e160d5 +Author: Julie Tibshirani +Date: Mon Sep 23 13:21:37 2019 -0700 + + Add support for aliases in queries on _index. (#46944) + + Previously, queries on the _index field were not able to specify index aliases. + This was a regression in functionality compared to the 'indices' query that was + deprecated and removed in 6.0. + + Now queries on _index can specify an alias, which is resolved to the concrete + index names when we check whether an index matches. To match a remote shard + target, the pattern needs to be of the form 'cluster:index' to match the + fully-qualified index name. Index aliases can be specified in the following query + types: term, terms, prefix, and wildcard. + +commit 08f28e642b9aa24a2c9be973782807f8ec47f4ce +Author: Jim Ferenczi +Date: Mon Sep 23 19:37:15 2019 +0200 + + Replace SearchContext with QueryShardContext in query builder tests (#46978) + + This commit replaces the SearchContext used in AbstractQueryTestCase with + a QueryShardContext in order to reduce the visibility of search contexts. + + Relates #46523 + +commit a61050378308ec892c823f8cb19e8f364371205c +Author: Costin Leau +Date: Mon Sep 23 18:59:46 2019 +0300 + + SQL: Add PIVOT support (#46489) + + Add initial PIVOT support for transforming a regular table into a + statistics table around an arbitrary pivoting column: + + SELECT * FROM + (SELECT languages, country, salary, FROM mp) + PIVOT (AVG(salary) FOR countries IN ('NL', 'DE', 'ES', 'RO', 'US')) + + In the current implementation PIVOT allows only one aggregation however + this restriction is likely to be lifted in the future. + Also not all aggregations are working, in particular MatrixStats are not yet supported. + + (cherry picked from commit d91263746a222915c570d4a662ec48c1d6b4f583) + +commit 618fb31be814012ae4ae98586b44a944b0dfb99d +Author: James Rodewig +Date: Mon Sep 23 13:23:26 2019 -0400 + + [DOCS] Minor editorial changes to enrich docs + +commit 199fff8a55fc31d23512abaf102a0270538e202d +Author: Eray +Date: Mon Sep 23 19:46:39 2019 +0300 + + Allow max_children only in top level nested sort (#46731) + + This commit restricts the usage of max_children to the top level nested sort since it is ignored on the other levels. + +commit a815f8b930255b03e69dc422163a1086ec869256 +Author: Lisa Cawley +Date: Mon Sep 23 08:45:01 2019 -0700 + + [DOCS] Group rollup and transform content (#46882) + +commit d4d1182677ce9c2715b4064fed5c34b73a8a5021 +Author: Luca Cavanna +Date: Mon Sep 23 17:00:37 2019 +0200 + + update _common.json format (#46872) + + API spec now use an object for the documentation field. _common was not updated yet. This commit updates _common.json and its corresponding parser. + + Closes #46744 + + Co-Authored-By: Tomas Della Vedova + +commit b09aba4c55eeb2a3a3c8cad1705d4d98b84f2ef1 +Author: James Rodewig +Date: Mon Sep 23 09:18:01 2019 -0400 + + [DOCS] Reformat rollover index API docs (#46778) + +commit 2da040601be092f3817317ad4cbe4a131e3f29db +Author: Armin Braun +Date: Mon Sep 23 15:01:47 2019 +0200 + + Fix Bug in Snapshot Status Response Timestamps (#46919) (#46970) + + Fixing a corner case where snapshot total time calculation was off when + getting the `SnapshotStatus` of an in-progress snapshot. + + Closes #46913 + +commit f06aa0c6c00d31578c44f0bdc122b132e76387df +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Sep 23 13:31:41 2019 +0200 + + Fix G1 GC default IHOP (#46169) + + G1 GC were setup to use an `InitiatingHeapOccupancyPercent` of 75. This + could leave used memory at a very high level for an extended duration, + triggering the real memory circuit breaker even at low activity levels. + The value is a threshold for old generation usage relative to total heap + size and thus it should leave room for the new generation. Default in + G1 is to allow up to 60 percent for new generation and this could mean that the + threshold was effectively at 135% heap usage. GC would still kick in of course and + eventually enough mixed collections would take place such that adaptive adjustment + of IHOP kicks in. + + The JVM has adaptive setting of the IHOP, but this does not kick in + until it has sampled a few collections. A newly started, relatively + quiet server with primarily new generation activity could thus + experience heap above 95% frequently for a duration. + + The changes here are two-fold: + 1. Use 30% default for IHOP (the JVM default of 45 could still mean + 105% heap usage threshold and did not fully ensure not to hit the + circuit breaker with low activity) + 2. Set G1ReservePercent=25. This is used by the adaptive IHOP mechanism, + meaning old/mixed GC should kick in no later than at 75% heap. This + ensures IHOP stays compatible with the real memory circuit breaker also + after being adjusted by adaptive IHOP. + +commit 5fd7505efc4d5e69804b49cf945e1eace9ae5e44 +Author: Alpar Torok +Date: Mon Sep 23 12:48:47 2019 +0300 + + Testfixtures allow a single service only (#46780) + + This PR adds some restrictions around testfixtures to make sure the same service ( as defiend in docker-compose.yml ) is not shared between multiple projects. + Sharing would break running with --parallel. + + Projects can still share fixtures as long as each has it;s own service within. + This is still useful to share some of the setup and configuration code of the fixture. + + Project now also have to specify a service name when calling useCluster to refer to a specific service. + If this is not the case all services will be claimed and the fixture can't be shared. + For this reason fixtures have to explicitly specify if they are using themselves ( fixture and tests in the same project ). + +commit ef0b75765b0820baa0afa4cd5a4e0117a49afce9 +Author: Mark Vieira +Date: Tue May 28 17:52:35 2019 -0700 + + Add explicit build flag for experimenting with test execution cacheability (#42649) + + * Add build flag for ignoring random test seed as task input + + * Fix checkstyle violations + +commit 7bc86f23ec394438b5f391315b1f4e116eac2063 +Author: David Turner +Date: Mon Sep 23 10:46:25 2019 +0100 + + Wait longer for leader failure in logs test (#46958) + + `testLogsWarningPeriodicallyIfClusterNotFormed` simulates a leader failure and + waits for long enough that a failing leader check is scheduled. However it does + not wait for the failing check to actually fail, which requires another two + actions and therefore might take up to 200ms more. Unlucky timing would result + in this test failing, for instance: + + ./gradle ':server:test' \ + --tests "org.elasticsearch.cluster.coordination.CoordinatorTests.testLogsWarningPeriodicallyIfClusterNotFormed" \ + -Dtests.jvm.argline="-Dhppc.bitmixer=DETERMINISTIC" \ + -Dtests.seed=F18CDD0EBEB5653:E9BC1A8B062E697A + + This commit adds the extra delay needed for the leader failure to complete as + expected. + + Fixes #46920 + +commit 084bda6d32411394879360b4e15fd3f4dbe98fab +Author: Martijn van Groningen +Date: Mon Sep 23 11:14:07 2019 +0200 + + fixed hlrc method signatures + +commit 33bbc4798b79a8e14fcdf03503a15c170acbde21 +Author: Martijn van Groningen +Date: Mon Sep 23 09:36:03 2019 +0200 + + fixed compile errors after merging + +commit 0cfddca61d0c0e062df3ee81702a90baa9918b23 +Merge: bf42789eb67 363beefbcde +Author: Martijn van Groningen +Date: Mon Sep 23 09:46:05 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 363beefbcde471f48df0fe3a0d398dee2f84d3d4 +Author: Martijn van Groningen +Date: Mon Sep 23 08:28:11 2019 +0200 + + Change HLRC count request to accept a QueryBuilder (#46904) + + Currently the CountRequest accepts a search source builder, while the + RestCountAction only accepts a top level query object. This can lead + to confusion if another element (e.g. aggregations) is specified, + because that will be ignored on the server side in RestCountAction. + + By deprecating the current setter & constructor that accept a + SearchSourceBuilder and adding replacement that accepts a QueryBuilder + it is clear what the count api can handle from HLRC side. + + Follow up from #46829 + +commit e8515d1d13ce841df912072cca4df90f8926a87f +Author: Nhat Nguyen +Date: Sun Sep 22 16:55:31 2019 -0400 + + Force flush in FullClusterRestartIT#testRecovery (#46956) + + If peer recovery happens after indexing, and indexing flushes some shard + at the end, then the explicit flush in the test will be a noop. Then + replicas will have some uncommitted translog , which is transferred in + peer recovery, although all of these operations are in the commit + already. If that replica becomes primary (after we restarted the + cluster), it will have translog to replay and the test will fail. + + Another issue in this test is that synced_flush is not a replication + action, then the global checkpoint on replicas might be not up to date. + We need to either wait for the global checkpoint to be synced or call a + replication action to sync it. + + Closes #46712 + +commit bf42789eb676c3a249cc10ebb0947d2ae7e6b812 +Author: Martijn van Groningen +Date: Sun Sep 22 21:20:12 2019 +0200 + + fixed compile error + +commit ee4e6b13828cf048fb6fd27c200ecc336815dfd7 +Author: Armin Braun +Date: Sat Sep 21 15:24:58 2019 +0200 + + Add TestLogging for #46701 (#46939) (#46949) + + This at a very low rate and with the force merge in place + before checking the cache size it's not clear why the cache + is not of size `0` -> seems something else + must be happening here that is unexpected. + -> add debug logging to this test to find out + + Relates #46701 + +commit 875d864be6bcf4d988a4d33e67cd4efc42efa25c +Author: Lisa Cawley +Date: Fri Sep 20 15:57:43 2019 -0700 + + [DOCS] Update data frame transform URLs (#46940) (#46946) + +commit f51b2592a465f62750ca92c2f5b64cfa2d66ecdf +Author: Mark Vieira +Date: Fri Sep 20 14:51:09 2019 -0700 + + Use composition instead of inheritance for extending Gradle plugins (#46888) + +commit a5661ac03ab7979a4868b8b76281ad8494c711c6 +Author: Javier Ruiz +Date: Fri Sep 20 21:19:54 2019 +0200 + + [DOCS] Fix calendar interval typos for date histo agg (#46911) + +commit 4c688df0ef96ecf0f1b8bf7830ed7942aea65be6 +Author: Lisa Cawley +Date: Fri Sep 20 11:31:24 2019 -0700 + + [DOCS] Add security section to Elasticsearch book (#46883) + + Co-Authored-By: Tim Vernum + +commit 99130114de8f8c0a657cbc9b480191b5fa760fdf +Author: James Rodewig +Date: Fri Sep 20 12:20:12 2019 -0400 + + [DOCS] Correct several [source,console-result] snippets (#46930) (#46937) + +commit f1c7ed647bb7b5000939b1f13318afdb16bd62cb +Author: Michael Basnight +Date: Tue Sep 10 10:42:28 2019 -0500 + + Allow comma separated ids in get enrich policy API (#46351) + + This commit changes the GET REST api so it will accept an optional comma + separated list of enrich policy ids. This change also modifies the + behavior of the GET API in that it will not error if it is passed a bad + enrich id anymore, but will instead just return an empty list. + +commit 0294b8fc6087de0f76fa053c84f933b139b34bf5 +Author: Alpar Torok +Date: Fri Sep 20 17:26:53 2019 +0300 + + Fix SymbolicLinkPreservingTarIT on windows (#46908) + + Broken path comparison on windows: + https://scans.gradle.com/s/7y3zl23orfp7w/tests/nr3ff3qzwqzsw-xfh7lvi5orvq4 + With the fix: + https://gradle-enterprise.elastic.co/s/4dcc2nv3haaqw + +commit 402fb5e8825ccdfcfe1194cf4e432410820b6f52 +Author: Andrei Dan +Date: Fri Sep 20 15:07:54 2019 +0100 + + Fix flaky test addSnapshotLifecyclePolicy (#46881) (#46912) + + * addSnapshotLifecyclePolicy drop version assertion + + This drops the assertion on the policy version (which was pinned to 1L) + as we want to execute both put policy apis (sync and async) for + documentation purposes. This will sometimes (depending on the async + call) yield a version of 2L. Waiting for the async call to always + complete could be an option but the test is already rather slow and it's + a bit of an overkill as we're already verifying the policy was created. + + (cherry picked from commit af4864c39129bcdbf98d00223f445346a62075e4) + Signed-off-by: Andrei Dan + +commit 938648fcffdf234a9178af5163d027a5562d7790 +Author: Armin Braun +Date: Fri Sep 20 14:30:03 2019 +0200 + + Remove Duplicate Shard Snapshot State Updates (#46862) (#46906) + + We were repeatedly trying to send shard state updates for aborted + snapshots on every cluster state update. + This is simply dead-code since those updates are already safely + sent in the callbacks passed to `SnapshotShardsService#snapshot`. + On master failover, we ensure that the status update is resent + via `SnapshotShardsService#syncShardStatsOnNewMaster`. + => there is no need for trying to send updates here over and over + and this logic can safely be removed + +commit 4a2cb05162f6e6f5053970872defa7281acd6340 +Author: Hendrik Muhs +Date: Fri Sep 20 13:00:45 2019 +0200 + + add message about transform disabled if license is missing (#46901) + + adds a message for transform about what happens if no license has been activated + +commit 97acf353faf5fe2a43f606c04147f380f39d73ac +Author: Jason Tedor +Date: Fri Sep 20 07:27:18 2019 -0400 + + Move pipelines resolved assertion (#46892) + + This assertion was added during the development of required + pipelines. In the initial version of that work, the notion of whether or + not a request was forwarded from the coordinating node to an ingest node + was introduced. It was realized later that instead we needed to track + whether or not the pipeline for the request was resolved. When that + change was made, this assertion, while not incorrect, was left behind + and only applied if the coordnating node was forwarding the + request. Instead, the assertion applies whether or not the request is + being forwarded. This commit addresses that by moving the assertion and + renaming some variables. + +commit fcea154f2ef9596e6fc9000633a9d246959892b4 +Author: István Zoltán Szabó +Date: Fri Sep 20 11:24:44 2019 +0200 + + [DOCS] Reformats Field capabilities API (#46866) + + * [DOCS] Reformats Field capabilities API. + Co-Authored-By: James Rodewig + +commit 363075cf1d1551593ffaeaaea184aee8dc3a49bd +Author: István Zoltán Szabó +Date: Fri Sep 20 10:59:11 2019 +0200 + + [DOCS] Reformats explain API (#46857) + + * [DOCS] Reformats explain API. + Co-Authored-By: James Rodewig + +commit abe889af754b1308b0c309ba63087d5a6a60124b +Author: Hendrik Muhs +Date: Fri Sep 20 10:43:00 2019 +0200 + + [7.5][Transform] rename classes in transform plugin (#46867) + + rename classes and settings in transform plugin, provide BWC for old settings + +commit d82712417f78bfa90ede75b8bf6f7dd12a8935ed +Author: Martijn van Groningen +Date: Fri Sep 20 09:57:35 2019 +0200 + + [HLRC] Send min_score as query string parameter to the count API (#46829) + + Prior to this commit min_score was sent as request body parameter + (via SearchSourceBuilder), which is not possible in the count api. + + Similar to #46474 + +commit add7148f3b3b002cadfd02a522a1cec7ce925887 +Author: Tanguy Leroux +Date: Fri Sep 20 09:52:50 2019 +0200 + + GCS deleteBlobsIgnoringIfNotExists should catch StorageException (#46832) + + GoogleCloudStorageBlobStore.deleteBlobsIgnoringIfNotExists() does + not correctly catch StorageException thrown by batch.submit(). + + In the case a snapshot is deleted through BlobStoreRepository.deleteSnapshot() + a storage exception is not caught (only IOException are) so the deletion is + interrupted and indices cannot be cleaned up. The storage exception bubbles + up to SnapshotService.deleteSnapshotFromRepository() but the listener that + removes the deletion from the cluster state is not executed, leaving the + deletion in the cluster state. + + This bug has been reported in #46772 where batch.submit() threw an + exception in the test testIndicesDeletedFromRepository and following + tests failed because a snapshot deletion was running. + + Relates #46772 + +commit 74d1588ec58156df0c45483c4ef29c0b5a2ce2b2 +Author: Colin Goodheart-Smithe +Date: Fri Sep 20 08:10:08 2019 +0100 + + Updates release notes for 7.4.0 + +commit 6ce48255948a4633a2ce519eee9d8c95d3530b2b +Author: Julie Tibshirani +Date: Thu Sep 19 20:26:52 2019 -0700 + + Add vector function changes to 7.4 migration docs. (#46893) + + This note was accidentally omitted in the breaking change #46004. + +commit 2425fd1a5088f2927a4da668aaca9556fe965ddf +Author: Jason Tedor +Date: Thu Sep 19 16:45:43 2019 -0400 + + Removed unused import from RequiredPipelineIT.java + + This commit removes an unused import that was left behind after cleaning + up a backport. Sorry. + +commit 6b00e00bc67df6378c1cf8f4257e3762c147cf17 +Author: rikardbakkehaug <45418376+rikardbakkehaug@users.noreply.github.com> +Date: Thu Sep 19 22:38:07 2019 +0200 + + [DOCS] Correct date math ex for `gt` and `gte` parms in `range` query docs (#46873) + +commit bd776261773ed2ba640c8f35f3803e2b946c62bc +Author: Jason Tedor +Date: Thu Sep 19 16:20:12 2019 -0400 + + Add the ability to require an ingest pipeline (#46847) + + This commit adds the ability to require an ingest pipeline on an + index. Today we can have a default pipeline, but that could be + overridden by a request pipeline parameter. This commit introduces a new + index setting index.required_pipeline that acts similarly to + index.default_pipeline, except that it can not be overridden by a + request pipeline parameter. Additionally, a default pipeline and a + request pipeline can not both be set. The required pipeline can be set + to _none to ensure that no pipeline ever runs for index requests on that + index. + +commit 251dbd8522e04080edc8edbfbb60841997c42f04 +Author: James Rodewig +Date: Thu Sep 19 15:56:24 2019 -0400 + + [DOCS] Remove `lowercase_terms` parm from term suggester docs (#46879) + +commit 8307aa3b71e2bb516af0e34ef31c5323cdf1b4c6 +Author: James Rodewig +Date: Thu Sep 19 15:23:52 2019 -0400 + + [DOCS] Update tagged region for index alias desc (#46886) + +commit c922743c5d5e62400303c81f577fdf0b430cf585 +Author: Armin Braun +Date: Thu Sep 19 18:52:35 2019 +0200 + + Remove Bogus Test: testDeleteOrphanSnapshot (#46835) (#46874) + + This test is broken with a very low failure rate after recent changes. + Particularly after #45689 which does not check for duplicate snapshot + uuids during snapshot finalization any more. + The check for duplicate uuids during finalization was removed + conciously since it lead to problems during master failover. + This test fails because it increments the repository state id in an + unexpected manner now, starting from the impossible situation of having + the same snapshot UUID for two different repository state ids. + This situation can't normally be reached, but we manually crafted it here. + + This test didn't do anything before though, because the manually crafted + cluster state would simply result in an error during finalization before + and nothing but a normal snapshot delete would be tested. + => removing this test here, it doesn't test anything. + + Closes #46843 + +commit 9638ca20b0ea25035eac561917efd629614bd3e0 +Author: Yannick Welsch +Date: Thu Sep 19 16:45:45 2019 +0200 + + Allow dropping documents with auto-generated ID (#46773) + + When using auto-generated IDs + the ingest drop processor (which looks to be used by filebeat + as well) + coordinating nodes that do not have the ingest processor functionality, this can lead + to a NullPointerException. + + The issue is that markCurrentItemAsDropped() is creating an UpdateResponse with no id when + the request contains auto-generated IDs. The response serialization is lenient for our + REST/XContent format (i.e. we will send "id" : null) but the internal transport format (used for + communication between nodes) assumes for this field to be non-null, which means that it can't + be serialized between nodes. Bulk requests with ingest functionality are processed on the + coordinating node if the node has the ingest capability, and only otherwise sent to a different + node. This means that, in order to reproduce this, one needs two nodes, with the coordinating + node not having the ingest functionality. + + Closes #46678 + +commit 7a3054c5dcf94d6d3b8bd0db91c8f7e629999470 +Author: Takumasa Ochi <4468155+aeroastro@users.noreply.github.com> +Date: Thu Sep 19 23:05:46 2019 +0900 + + Fix typos in `match` in profile API (#46723) + + * Replace `matches` with correct `match` + * Use present tense consistently + * Replace `metric` with correct `match` + +commit b32031ee03ab9de729be8c4c3494c17162ed15f2 +Author: Armin Braun +Date: Thu Sep 19 16:05:51 2019 +0200 + + Fix snapshot_restore_exception in Docs Tests (#46852) (#46870) + + We should only snapshot the index we're going to + restore in the next step. Otherwise, we will + potentially not get the correct response or + fail restoring outright due to internal indices + getting created concurrently when running against + the x-pack distribution. + + Closes #46844 + +commit 4a9943a906052a419551d19ebdacbb8f2bfdc8a4 +Author: Jason Tedor +Date: Thu Sep 19 05:55:39 2019 -0700 + + Bundle AdoptOpenJDK 13 (#46860) + + This commit switches to use AdoptOpenJDK 13 for bundling the JDK, + instead of Oracle OpenJDK 13. + +commit e467dddd4c1d20e149194155dbf30eed339ab5b7 +Author: Russ Cam +Date: Thu Sep 19 22:26:31 2019 +1000 + + Use snake casing for document field (#45432) + + This commit updates the search example to use snake casing for the document field. This is consistent with other examples + +commit f1de8c019355b9ebf7b9511308f6a1c3f97827ba +Author: Martijn Laarman +Date: Thu Sep 19 14:23:12 2019 +0200 + + Document PUT as the prefered HTTP method for index API (#45622) + + Follow up from #42346. Since the `methods` array is in order of + preference when calling the index API with an `{id}` we prefer to use + the `PUT` http method. + +commit b73a9604c1db99c88466d9b843fe63113ed91cfa +Author: James Rodewig +Date: Thu Sep 19 08:22:37 2019 -0400 + + [DOCS] Separate and reformat synced flush API docs (#46634) (#46839) + +commit e555ef2e86d39991091a74f8d3ffc660cf375d60 +Author: LHearen +Date: Thu Sep 19 20:20:11 2019 +0800 + + [DOCS] make bulk page more readable (#46200) + + comma and semi-colon required to make it more readable + +commit 0f2e72b7f8a68c67ffead685b7b515d01b32f3b0 +Author: maidoo +Date: Thu Sep 19 20:14:35 2019 +0800 + + [DOCS] Term dict memory usage is a 7.3.0 enhancement, not 7.2.0 (#46662) + + Closes #46635 + +commit 7795e21ea3ed1a053b2883af7ac354674c43751e +Author: Toby McLaughlin +Date: Thu Sep 19 22:05:51 2019 +1000 + + [DOCS] Fix typo in GCS snapshot repo doc (#46048) + +commit 6b09c2cdbb2dee6b1a5e1168f35ef68f9ce9ae91 +Author: Armin Braun +Date: Thu Sep 19 13:07:42 2019 +0200 + + Limit Netty Workers in NativeRealmIntegTestCase (#46816) (#46850) + + The fact that this test randomly uses a relatively large number + of nodes and hence Netty worker threads created a problem with + running out of direct memory on CI. + Tests run with 512M heap (and hence 512M direct memory) by default. + On a CI worker with 16 cores, this means Netty will by default set + up 32 transport workers. If we get unlucky and a lot of them + actually do work (and thus instantiate a `CopyBytesSocketChannel` + which costs 1M per thread for the thread-local IO buffer) we + would run out of memory. + + This specific failure was only seen with `NativeRealmIntegTests` so I + only added the constraint on the Netty worker count here. + We can add it to other tests (or `SecurityIntegTestCase`) if need be + but for now it doesn't seem necessary so I opted for least impact. + + Closes #46803 + +commit a08755300915879c2e1b8d8c31876cb0549dc6d9 +Author: Armin Braun +Date: Thu Sep 19 13:07:00 2019 +0200 + + Rearrange BlobStoreRepository to Prepare #46250 (#46824) (#46853) + + In #46250 we will have to move to two different delete + paths for BwC. Both paths will share the initial + reads from the repository though so I extracted/separated the + delete logic away from the initial reads to significantly simplify + the diff against #46250 here. + Also, I added some JavaDoc from #46250 here as well which makes the + code a little easier to follow even ignoring #46250 I think. + +commit a6c9f8ffae09d369af7e72e7fd8aa40dacede724 +Author: Luca Cavanna +Date: Thu Sep 19 09:34:52 2019 +0200 + + [DOCS] mention automatic cancellation in search docs (#46706) + + Add a small note around automatic search cancellation which was missed as part of #43332 + + Relates to #43332 + +commit 3be51fbdf7a8ba151b6615a83881a9e285fc0a60 +Author: István Zoltán Szabó +Date: Thu Sep 19 09:10:11 2019 +0200 + + [DOCS] Adds regression analytics resources and examples to the data frame analytics APIs and the evaluation API (#46176) + + * [DOCS] Adds regression analytics resources and examples to the data frame analytics APIs. + Co-Authored-By: Benjamin Trent + Co-Authored-By: Tom Veasey + +commit 65fffcc9c1600f64bd97c489f6ed2a0c0a867ee9 +Author: István Zoltán Szabó +Date: Thu Sep 19 09:03:37 2019 +0200 + + [DOCS]Changes failed transforms limitation. (#46754) + +commit 86d216cc9cd7f2414906194f4a41b1343719d780 +Author: Alpar Torok +Date: Thu Sep 19 09:35:01 2019 +0300 + + Update .ci/bwcVersions + +commit a32fe1618cff7a70f2cd77afe511718219b3199b +Author: Lisa Cawley +Date: Wed Sep 18 13:30:16 2019 -0700 + + [DOCS] Updates transform landing page (#46689) + +commit 507d879b70f0e9f2cd6d54e30e696507c78aef27 +Author: Mark Vieira +Date: Wed Sep 18 11:51:44 2019 -0700 + + Disable bwc distribution caching in master branch (#46686) + + This commit disables caching of BWC snapshot distributions in the "trunk" (aka master) branch. + Since the previous major release branches move quickly we rarely get cache hits for these + tasks, and the artifacts themselves are very large. This means the overhead here is high and + savings basically zero. We conditionally disable task output caching in this scenario in CI to + avoid excessive build cache overhead as well as causing too much turn in the cache itself which + would lead to lots of cache entry evictions. + +commit 02a5e153dc7cabb19663f0647b07bdf82d229ce4 +Author: Dimitris Athanasiou +Date: Wed Sep 18 20:37:40 2019 +0300 + + [7.x][ML] Parse and index data frame analytics state (#46804) (#46820) + + This commit reuses the same state processor that is used for autodetect + to parse state output from data frame analytics jobs. We then index the + state document into the state index. + + Backport of #46804 + +commit b08b3741db225109ad1882afb4d096c553c78c3d +Author: James Rodewig +Date: Wed Sep 18 11:08:45 2019 -0400 + + [DOCS] Use versions file from elastic/docs (#46719) + +commit d00bad9ab27344e9caccba6b9f13a14158519a5a +Author: James Rodewig +Date: Wed Sep 18 10:10:16 2019 -0400 + + [DOCS] Reformat force merge API docs (#46685) + +commit 9cf9c64ec2fa6e4e6adfe940270a552960fbdf83 +Author: Benjamin Trent +Date: Wed Sep 18 10:06:05 2019 -0400 + + [7.x] [ML][Transforms] remove `force` flag from _start (#46414) (#46748) + + * [ML][Transforms] remove `force` flag from _start (#46414) + + * [ML][Transforms] remove `force` flag from _start + + * fixing expected error message + + * adjusting bwc version + +commit d1f47cf00e7602c2974586d3f2a737a4ab7a42b4 +Author: Turaç Kangal +Date: Wed Sep 18 16:49:38 2019 +0300 + + [HLRC] Send terminate_after as query string parameter to the count API (#46474) + + Prior to this commit terminate_after was sent as request body parameter + (via SearchSourceBuilder), which is not possible in the count api. + + Closes #46446 + +commit 6863b9027244c06938a8e9f50e7d8a248e33a996 +Author: James Rodewig +Date: Wed Sep 18 09:44:28 2019 -0400 + + [DOCS] Remove leveloffset for CCR docs (#46818) + +commit 3ae51f25dd4d8f7eefc03b9b52aee8701f6d3051 +Author: Tanguy Leroux +Date: Wed Sep 18 15:30:51 2019 +0200 + + Move testSnapshotWithLargeSegmentFiles to ESMockAPIBasedRepositoryIntegTestCase (#46802) + + This commit moves the common test testSnapshotWithLargeSegmentFiles + to the ESMockAPIBasedRepositoryIntegTestCase base class. + +commit cebe8da61789e5fac9d399c515bd1d9063fa1cba +Author: Dimitris Athanasiou +Date: Wed Sep 18 16:35:25 2019 +0300 + + [7.x][ML] MlMemoryTracker should ignore analytics tasks without config (#46789) (#46811) + + It is possible for a running analytics job that its config is removed + from the '.ml-config' index (perhaps the user deleted the entire index, + etc.). In that case the task remains without a matching config. I have + raised #46781 to discuss how to deal with this issue. + + This commit focuses on `MlMemoryTracker` and changes it so that when + we get the configs for the running tasks we leniently ignore missing ones. + This at least means memory tracking will keep working for other jobs + if one or more are missing. + + In addition, this commit makes the cleanup code for native analytics + tests more robust by explicitly stopping all jobs and force-stopping + if an error occurs. This helps so that a single failing test does + not cause other tests fail due to pending tasks. + + Backport of #46789 + +commit 0076083b359e19a007c03fac562ec2db65fb966a +Author: Christos Soulios <1561376+csoulios@users.noreply.github.com> +Date: Wed Sep 18 15:56:34 2019 +0300 + + Implement rounding optimization for fixed offset timezones (#46809) + + Fixes #45702 with date_histogram aggregation when using fixed_interval. + Optimization has been implemented for both fixed and calendar intervals + +commit e59be0354acdb265b8382ae7d900dfcf5f975571 +Author: István Zoltán Szabó +Date: Wed Sep 18 14:29:48 2019 +0200 + + [DOCS] Reformats validate API (#46389) + + * [DOCS] Reformats validate API. + Co-Authored-By: James Rodewig + +commit b6da5fa2f76553ff5f3d8e8343f4a05cdba6c1c0 (es/enrich-7.x) +Author: James Rodewig +Date: Wed Sep 18 08:30:22 2019 -0400 + + [DOCS] Correct `` parm description for comma-sep list (#46682) + +commit 142b10604eac2161cfd5be6d94c2e64a662fa635 +Author: Armin Braun +Date: Wed Sep 18 13:22:55 2019 +0200 + + Fix testHistoryRetention (#46799) (#46805) + + Suppress the reasonable-history check in this test to guarantee we're always getting ops based recovery even after a background sync. + + Closes #45953 + + Co-Authored-By: David Turner + +commit f983b67fdcb1e0711fb95ec3668c18dacebaa2d9 +Author: Armin Braun +Date: Wed Sep 18 13:15:56 2019 +0200 + + Add Assertion About Leaking index-N to Repo Tests (#46774) (#46801) + + This adds an assert to make sure we're not leaking + index-N blobs on the shard level to the repo consistency checks. + It is ok to have a single redundant index-N blob in a failure scenario + but additional index-N should always be cleaned up before adding more. + +commit f3e67bdd17dd9b8459e8ec3d641bfe1c7e844395 +Author: Alpar Torok +Date: Wed Sep 18 10:39:48 2019 +0300 + + Add resolution rule to allow resolving all deps (#46768) + + Since the `resolveAllDependencies` task resolves all the congfigurations + it can find, this was not caught by our testing, but it's required to be + configuraed specifically. + + We should probably cut-over to the new configurations at some point to + avoid problems like this. + + Closes elastic/infra#14580 + +commit 799f7def9fc5e96f1b4f56befbc747d41a055d53 +Author: Tanguy Leroux +Date: Wed Sep 18 09:42:16 2019 +0200 + + Add block support to AzureBlobStoreRepositoryTests (#46664) + + This commit adds support for Put Block API to the internal HTTP server + used in Azure repository integration tests. This allows to test the + behavior of the Azure SDK client when the Azure Storage service + returns errors when uploading Blob in multiple blocks or when + downloading a blob using ranged downloads. + +commit fd42358a6d394e4ead3b77133ef50edac532bf95 +Author: Tanguy Leroux +Date: Wed Sep 18 09:39:17 2019 +0200 + + Add support for Multipart upload to S3 repository integration tests (#46704) + + This commit adds support for Multipart upload to the internal HTTP + server used in S3 repository integration tests. + +commit 4db37801d0923c9a8f3a48195962027860926e94 +Author: Tanguy Leroux +Date: Wed Sep 18 09:32:14 2019 +0200 + + Add resumable uploads support to GCS repository integration tests (#46562) + + This commit adds support for resumable uploads to the internal HTTP + server used in GoogleCloudStorageBlobStoreRepositoryTests. This + way we can also test the behavior of the Google's client when the + service returns server errors in response to resumable upload requests. + + The BlobStore implementation for GCS has the choice between 2 + methods to upload a blob: resumable and multipart. In the current + implementation, the client executes a resumable upload if the blob + size is larger than LARGE_BLOB_THRESHOLD_BYTE_SIZE, + otherwise it executes a multipart upload. This commit makes this + logic overridable in tests, allowing to randomize the decision of + using one method or the other. + + The commit add support for single request resumable uploads + and chunked resumable uploads (the blob is uploaded into multiple + 2Mb chunks; each chunk being a resumable upload). For this last + case, this PR also adds a test testSnapshotWithLargeSegmentFiles + which makes it more probable that a chunked resumable upload is + executed. + +commit d931665f91f513d9bdbb62d8d9b1ebf250464132 +Author: István Zoltán Szabó +Date: Wed Sep 18 09:14:32 2019 +0200 + + [DOCS] Adds force parameter to Stop transforms API docs. (#46770) + +commit ac4e99092493659b41b71c197db3a45988651ef5 +Author: Martijn van Groningen +Date: Wed Sep 18 08:45:15 2019 +0200 + + Add ingest cluster state listeners (#46650) + + In the case that an ingest processor factory relies on other configuration + in the cluster state in order to construct a processor instance then + it is currently undetermined if a processor facotry can be notified about + a change if multiple cluster state updates are bundled together and + if a processor implement `ClusterStateApplier` interface. + (IngestService implements this interface too) + + The idea with ingest cluster state listener is that it is guaranteed to + update the processor factory first before the ingest service creates + a pipeline with their respective processor instances. + + Currently this concept is used in the enrich branch: + https://github.com/elastic/elasticsearch/blob/enrich/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java#L21 + + In this case it a processor factory is interested in enrich indices' _meta + mapping fields. + + This is the third PR that merges changes made to server module from + the enrich branch (see #32789) into the master branch. + + Changes to the server module are merged separately from the pr that will + merge enrich into master, so that these changes can be reviewed in isolation. + +commit 7e71b2dbb35540c95910042a3c3d7eeab0653f92 +Author: Jason Tedor +Date: Tue Sep 17 17:50:52 2019 -0700 + + Update bundled JDK to JDK 13 (#46795) + + With the next minor release of Elasticsearch we will drop support for + JDK 12 and bump to JDK 13. While we want to use AdoptOpenJDK as the + bundled JDK, we are waiting for a release there. This commit moves to + OpenJDK 13 for now, and we will move to AdoptOpenJDK 13 as soon as its + available. Since macOS Catalina is delayed until October, we have some + time to update this. + +commit b85468d6ea407aa7133400844267fe287e92539c +Author: Lee Hinman +Date: Tue Sep 17 17:39:41 2019 -0600 + + Add node setting for disabling SLM (#46794) (#46796) + + This adds the `xpack.slm.enabled` setting to allow disabling of SLM + functionality as well as its HTTP API endpoints. + + Relates to #38461 + +commit cbd58d3b78ad037750073ae31c0409382afc0d05 +Author: Oliver Gupte +Date: Tue Sep 17 15:01:42 2019 -0700 + + Give kibana user privileges to create APM agent config index (#46765) (#46792) + + * Give kibana user reserved role privileges on .apm-* to create APM agent configuration index. + + * fixed test to include checking all .apm-* permissions + + * changed pattern from ".apm-*" to the more specific ".apm-agent-configuration" + +commit cd71d4a83b50fb5125e114e9411f2af4a0aa5052 +Author: Jason Tedor +Date: Tue Sep 17 13:40:35 2019 -0700 + + Use AdoptOpenJDK as the bundled JDK (#46470) (#46785) + + This commit teaches the build how to bundle AdoptOpenJDK with our + artifacts, and switches to AdoptOpenJDK as the bundled JDK. We keep the + functionality to also bundle Oracle OpenJDK distributions. + +commit 00c604a5a61f3c1f318177b630781a52b156338a +Author: Lisa Cawley +Date: Tue Sep 17 11:44:16 2019 -0700 + + [DOCS] Augments ecommerce example (#46788) + +commit 9dcf8a53a2d06726a6fab7da54c421876b888eb3 +Author: James Rodewig +Date: Tue Sep 17 14:25:46 2019 -0400 + + [DOCS] Add CCR docs to the Elasticsearch Reference Guide (#45704) + + Previously, cross-cluster replication (CCR) documentation was located in + the Stack Overview: + https://www.elastic.co/guide/en/elastic-stack-overview/master/xpack-ccr.html + + This adds CCR documentation to the Elasticsearch Reference Guide with a + level offset for headings. + + The level offset and CCR Stack Overview docs will be removed in later + commits. + +commit 371c355bca444f5bfaf2775c8125593c525202e0 +Author: Armin Braun +Date: Tue Sep 17 19:06:43 2019 +0200 + + Retry GCS Resumable Upload on Error 410 (#45963) (#46783) + + A resumable upload session can fail on with a 410 error and should + be retried in that case. I added retrying twice using resetting of + the given `InputStream` as the retry mechanism since the same + approach is used by the AWS S3 SDK already as well and relied upon + by the S3 repository implementation. + + Related GCS documentation: + https://cloud.google.com/storage/docs/json_api/v1/status-codes#410_Gone + +commit 2c70d403fc06e9ebaa71378efdf876865bb3a98f +Author: Armin Braun +Date: Tue Sep 17 18:09:48 2019 +0200 + + Reenable+Fix testMasterShutdownDuringFailedSnapshot (#46303) (#46747) + + Reenable this test since it was fixed by #45689 in production + code (specifically, the fact that we write the `snap-` blobs + without overwrite checks now). + Only required adding the assumed blocking on index file writes + to test code to properly work again. + + * Closes #25281 + +commit b00de8edf3708aff74093cba1996d53428b0a0d3 +Author: Armin Braun +Date: Tue Sep 17 15:40:11 2019 +0200 + + Ensure SAS Tokens in Test Use Minimal Permissions (#46112) (#46628) + + There were some issues with the Azure implementation requiring + permissions to list all containers ue to a container exists + check. This was caught in CI this time, but going forward we + should ensure that CI is executed using a token that does not + allow listing containers. + + Relates #43288 + +commit 92e518e789b4d903f9eb3cade1d299e4d97dac76 +Author: Costin Leau +Date: Tue Sep 17 15:29:23 2019 +0300 + + SQL: Properly handle indices with no/empty mapping (#46775) + + When encountering only indices with empty mapping, the IndexResolver + throws an exception as it expects to find at least one entry. + This commit fixes this case so that an empty mapping is returned. + + Fix #46757 + + (cherry picked from commit 5f4f5807acb93b5fab36718c092c328977a396b6) + +commit 65dc88862349b97da96eee9796fdfa6ddec0630f +Author: David Turner +Date: Tue Sep 17 13:10:55 2019 +0100 + + Resume partial download from S3 on connection drop (#46589) + + Today if the connection to S3 times out or drops after starting to download an + object then the SDK does not attempt to recover or resume the download, causing + the restore of the whole shard to fail and retry. This commit allows + Elasticsearch to detect such a mid-stream failure and to resume the download + from where it failed. + +commit b0f09b279f6289ff28d1b5c812135a10bd3a4a61 +Author: Armin Braun +Date: Tue Sep 17 13:09:39 2019 +0200 + + Make Snapshot Logic Write Metadata after Segments (#45689) (#46764) + + * Write metadata during snapshot finalization after segment files to prevent outdated metadata in case of dynamic mapping updates as explained in #41581 + * Keep the old behavior of writing the metadata beforehand in the case of mixed version clusters for BwC reasons + * Still overwrite the metadata in the end, so even a mixed version cluster is fixed by this change if a newer version master does the finalization + * Fixes #41581 + +commit e1cf1039805e6cf7d8c31d63a0916726f7bc3502 +Author: Tomas Della Vedova +Date: Tue Sep 17 11:49:24 2019 +0200 + + Fixes for API specification (#46522) (#46736) + + Follow-up of #42346 + +commit 683b5fdecafc631737068dfadd393e6fb701cae8 +Author: Costin Leau +Date: Tue Sep 17 11:04:45 2019 +0300 + + SQL: Support queries with HAVING over SELECT (#46709) + + Handle queries with implicit GROUP BY where the aggregation is not in + the projection/SELECT but inside the filter/HAVING such as: + + SELECT 1 FROM x HAVING COUNT(*) > 0 + + The engine now properly identifies the case and handles it accordingly. + + Fix #37051 + + (cherry picked from commit fa53ca05d8219c27079b50b4a5b7aeb220c7cde2) + +commit 90f4c2379b59d02d9f31468decf80a9081915093 +Author: Costin Leau +Date: Tue Sep 17 10:41:49 2019 +0300 + + SQL: improve ResultSet behavior when no rows are available (#46753) + + Improve the defensive behavior of ResultSet when dealing with incorrect + API usage. In particular handle the case of dealing with no row + available (either because the cursor is before the first entry or after + the last). + + Fix #46750 + + (cherry picked from commit 58fa38e4606625962e879265d35eacb0960c6cdb) + +commit 9d6632d6fa98ba08d52fb2aae40153f6b3d12900 +Author: Colin Goodheart-Smithe +Date: Tue Sep 17 09:06:46 2019 +0100 + + Updates 7.4.0 release notes + +commit 595bf52927f98405de4c47796d6dfbbdf1312c81 +Author: István Zoltán Szabó +Date: Tue Sep 17 09:53:03 2019 +0200 + + [DOCS] Reformats count API (#46377) + + * [DOCS] Reformats count API. + Co-Authored-By: James Rodewig + +commit c045bc7f54fe83c31c5fc571bddf8946d15d3551 +Author: Armin Braun +Date: Tue Sep 17 09:23:00 2019 +0200 + + Minor Rearrangements in Snapshot Code (#46652) (#46752) + + Inlining one trivial single-use method and extracting the + stale shard path blob calculation to make the diff with #46250 + more manageable. + +commit e49be611ad67a1565e7e82814a66cf3d5b58bc40 +Author: Przemysław Witek +Date: Mon Sep 16 21:21:38 2019 +0200 + + [7.x] Add audit messages for Data Frame Analytics (#46521) (#46738) + +commit a9eb538e9335ca7d7abbd54afc9f9e0a496372a8 +Author: Lisa Cawley +Date: Mon Sep 16 10:23:46 2019 -0700 + + [DOCS] Removes data-frames folder (#46760) + +commit d62e1a3d8b8b78884f03d3713c9d14f010429fca +Author: Lisa Cawley +Date: Mon Sep 16 10:00:44 2019 -0700 + + [DOCS] Fixes data frame analytics job terminology in HLRC (#46758) + +commit 48099cb18c9fe998ebb800cb73d7b6898469d629 +Author: lcawl +Date: Mon Sep 16 10:01:54 2019 -0700 + + [DOCS] Fixes formatting of analyzer parameter + +commit dddc9b3d739b2948ea117d95f439bb345fe20986 +Author: Lisa Cawley +Date: Mon Sep 16 08:28:19 2019 -0700 + + [DOCS] Updates dataframe transform terminology (#46642) + +commit 92acc732de3bb872272b2ede83dbb09de78604b7 +Author: Benjamin Trent +Date: Mon Sep 16 10:05:55 2019 -0400 + + [ML][Transform] Use field caps for mapping deductino (#46703) (#46742) + +commit 20cb95ca5e811a28817f6bd026ccaee599767617 +Author: Armin Braun +Date: Mon Sep 16 15:15:33 2019 +0200 + + Fix testSnapshotRelocatingPrimary to Actually Run Relocations (#46594) (#46620) + + Without replicas we won't actually get any relocations + going when removing the node constraints in this test. + Adjusted the code to force relocations by forbidding + nodes that hold primaries instead. + Also, fixed the timeouts and asserted that we actually + get relocations. + + Fixes #46276 + +commit 40e93539475bfe5f51e3d65f4179a0a718c86134 +Author: Andrei Stefan +Date: Mon Sep 16 15:31:37 2019 +0300 + + SQL: use the correct data type for types conversion (#46574) + + (cherry picked from commit 3e25db2f302c3aafe27e4d8d4fb1743401d85e6d) + +commit fe8f33a8e1955ed4f16a973cd2ebd1096cb6c2f0 +Author: István Zoltán Szabó +Date: Mon Sep 16 14:21:50 2019 +0200 + + [DOCS] Adds outlier detection params to the data frame analytics resources (#46323) + + * [DOCS] Adds outlier detection params to the data frame analytics resources. + Co-Authored-By: Tom Veasey + Co-Authored-By: Lisa Cawley + +commit c8f52ec4ff5660d231dbe9bb252039a3e96d8b53 +Author: Hendrik Muhs +Date: Mon Sep 16 13:39:22 2019 +0200 + + [Transform] Rename data frame plugin to transform: classes in xpack.core (#46644) (#46734) + + rename classes in xpack.core of transform plugin from "data frame transform" to "transform" + +commit 16bed7f017f00fcbe466f6ded8b8a9f544c4cc02 +Author: Costin Leau +Date: Mon Sep 16 12:32:56 2019 +0300 + + DOC: Fix typo + + (cherry picked from commit d874f1139ef0335db181d932f1e1299e1c8e40b8) + +commit 011496ed5fabe3516607a663123d90bfbdb2cdd6 +Author: Alexander Reelsen +Date: Mon Sep 16 11:29:59 2019 +0200 + + Expose cache setting in UserAgentPlugin (#46533) + + The setting was not registered. Also documentation has been added. + +commit e5eb4b49c938639c3cf68c95fb933869b208c527 +Author: Costin Leau +Date: Mon Sep 16 12:26:48 2019 +0300 + + DOC: Update section for SQuirrel SQL 4.0.0 (#46726) + + The just released SQuirrel SQL 4.0.0 provides an Elasticsearch driver + definition out of the box; update the documentation to reflect that. + + (cherry picked from commit 3aa417ed74947e69f0ff605b1c210a0678a3cb9f) + +commit c57cca98b2fd288f45760249133023de3d848b11 +Author: Andrei Dan +Date: Mon Sep 16 08:50:28 2019 +0100 + + [ILM] Add date setting to calculate index age (#46561) (#46697) + + * [ILM] Add date setting to calculate index age + + Add the `index.lifecycle.origination_date` to allow users to configure a + custom date that'll be used to calculate the index age for the phase + transmissions (as opposed to the default index creation date). + + This could be useful for users to create an index with an "older" + origination date when indexing old data. + + Relates to #42449. + + * [ILM] Don't override creation date on policy init + + The initial approach we took was to override the lifecycle creation date + if the `index.lifecycle.origination_date` setting was set. This had the + disadvantage of the user not being able to update the `origination_date` + anymore once set. + + This commit changes the way we makes use of the + `index.lifecycle.origination_date` setting by checking its value when + we calculate the index age (ie. at "read time") and, in case it's not + set, default to the index creation date. + + * Make origination date setting index scope dynamic + + * Document orignation date setting in ilm settings + + (cherry picked from commit d5bd2bb77ee28c1978ab6679f941d7c02e389d32) + Signed-off-by: Andrei Dan + +commit c99adfb99a027bfce89770c1d4afc927ee19ac70 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Sep 16 09:29:18 2019 +0200 + + Relax ReindexIT test timeouts (#46312) + + A couple of tests used 2 second timeout, which was prone to failure, + increased to 10 seconds. + + Closes #46301 + +commit 2b85dcb2013a1a80c3edb363323f2c96f47b9612 +Author: Armin Braun +Date: Mon Sep 16 07:52:03 2019 +0200 + + Parallelize Repository Cleanup Actions (#46647) (#46714) + + * Parallelize Repository Cleanup Actions + + Deleting root blobs and unreferenced indices can safely happen in parallel, + no need to have both operations run sequentially when they preclude all other + repository operations. + +commit 63eb0d9081d21b148874083bc61ae5b602cb8515 +Author: Dimitris Athanasiou +Date: Sun Sep 15 17:25:26 2019 +0300 + + [7.x][ML] Avoid marking data frame analytics task completed twice (#46721) (#46724) + + When the stop API is called while the task is running there is + a chance the task gets marked completed twice. This may cause + undesired side effects, like indexing the progress document a second + time after the stop API has returned (the cause for #46705). + + This commit adds a check that the task has not been completed before + proceeding to mark it so. In addition, when we update the task's state + we could get some warnings that the task was missing if the stop API + has been called in the meantime. We now check the errors are + `ResourceNotFoundException` and ignore them if so. + + Closes #46705 + + Backports #46721 + +commit e1842c0e5ad045dfe2df2074be449a07adce4b2b +Author: Hendrik Muhs +Date: Sat Sep 14 13:06:48 2019 +0200 + + [7.x][Transforms] backport BWC tests for transforms crud (#46452) + + backport 8.0 transform tests to 7.x + +commit 9774377959a197133e8249f9d06f96f7ac46a62b +Author: Mark Vieira +Date: Fri Sep 13 16:07:19 2019 -0700 + + Log actual checkout hash instead of generic refspec (#46637) + +commit 272b0ecbddf66724826254ceac953344595ed734 +Author: David Turner +Date: Fri Sep 13 22:17:03 2019 +0100 + + Remove docs for proxy mode (#46677) + + We added docs for proxy mode in #40281 but on reflection we should not be + documenting this setting since it does not play well with all proxies and we + can't recommend its use. This commit removes those docs and expands its Javadoc + instead. + +commit c0a16047fa6a7b85937bdea8046cc4999f1bc467 +Author: Lisa Cawley +Date: Fri Sep 13 11:38:51 2019 -0700 + + [DOCS] Updates links to reporting content (#46717) + +commit 2831535cf913ec4ee74acc98a78dd57c60e109d7 +Author: James Rodewig +Date: Fri Sep 13 11:23:53 2019 -0400 + + [DOCS] Replace "// CONSOLE" comments with [source,console] (#46679) + +commit 0def429bc1ac8b57f4a3cc8a29713d01dc653b38 +Author: James Rodewig +Date: Fri Sep 13 09:45:32 2019 -0400 + + [DOCS] Correct anchors and title for Update index settings API docs (#46707) + +commit cabff5a7cd241002f85d462166cf4403cc38653e +Author: Nhat Nguyen +Date: Thu Sep 12 16:20:27 2019 -0400 + + Handle lower retaining seqno retention lease error (#46420) + + We renew the CCR retention lease at a fixed interval, therefore it's + possible to have more than one in-flight renewal requests at the same + time. If requests arrive out of order, then the assertion is violated. + + Closes #46416 + Closes #46013 + +commit 0bc8acaf5bd520f7678ffaf1279737bd6a6cabe4 +Author: Dimitris Athanasiou +Date: Fri Sep 13 10:34:12 2019 +0300 + + [7.x][ML] Create state index and alias before starting an analytics job (#46602) (#46648) + + This is fixing a bug where if an analytics job is started before any + anomaly detection job is opened, we create an index after the state + write alias. + + Instead, we should create the state index and alias before starting + an analytics job and this commit makes sure this is the case. + + Backport of #46602 + +commit dae5b22bf861ce41a58f42527c12af56d2ee8938 +Author: Lisa Cawley +Date: Thu Sep 12 16:28:40 2019 -0700 + + [DOCS] Fixes link to Kibana security (#46690) + +commit e1a33c62833ebf18d6825e4ef073aa0322528e64 +Author: Nhat Nguyen +Date: Wed Sep 11 10:50:27 2019 -0400 + + Fix false positive out of sync warning in synced-flush (#46576) + + Synced-flush consists of three steps: (1) force-flush on every active + copy; (2) check for ongoing indexing operations; (3) seal copies if + there's no change since step 1. If some indexing operations are + completed on the primary but not replicas, then Lucene commits from step + 1 on replicas won't be the same as the primary's. And step 2 would pass + if it's executed when all pending operations are done. Once step 2 + passes, we will incorrectly emit the "out of sync" warning message + although nothing wrong here. + + Relates #28464 + Relates #30244 + +commit 5465c8d095cdc4e084dc560b1b76720e0fa3a8a4 +Author: Nhat Nguyen +Date: Wed Sep 11 10:34:14 2019 -0400 + + Increase timeout for relocation tests (#46554) + + There's nothing wrong in the logs from these failures. I think 30 + seconds might not be enough to relocate shards with many documents as CI + is quite slow. This change increases the timeout to 60 seconds for these + relocation tests. It also dumps the hot threads in case of timed out. + + Closes #46526 + Closes #46439 + +commit 5b1f6669ff9e6568841af491eeed0792b014093a +Author: Przemysław Witek +Date: Thu Sep 12 21:47:25 2019 +0200 + + Do not wait for the old notifications index (".ml-notifications"). It is no longer used. (#46657) (#46666) + +commit e1e06c2589e7c7ddf5e83182542010e97dfae274 +Author: Zachary Tong +Date: Thu Sep 12 13:50:40 2019 -0400 + + Add version constant for 7.3.3 + +commit af7aba18d489c3584a73f78a82f053d8f7bff489 +Author: James Rodewig +Date: Thu Sep 12 13:29:07 2019 -0400 + + Fixed sample code for minhash (#46385) + + The sample code is wrong. Field type is required for the sample field. + I guess the intention was to give the sample field the name ```fingerprint```, mapping it as ```text``` using the custom analyzer ```my_analyzer``` + +commit 3b2665733500c45f7812a5d9319361fcf24b5f5e +Author: James Rodewig +Date: Thu Sep 12 13:25:41 2019 -0400 + + [DOCS] Remove coming tags for 7.3.2 release (#46676) + +commit 411d4e9a938f34170b4d44c8ffab3b83a4f95372 +Author: James Rodewig +Date: Thu Sep 12 10:13:21 2019 -0400 + + [DOCS] Change // CONSOLE comments to [source,console] (#46669) + +commit 9229b87bd22d1a918ab2026263272e9bd373572a +Author: Alpar Torok +Date: Thu Sep 12 16:04:09 2019 +0300 + + Upgrade to Gradle 5.6 (#45005) + + * Upgrade to Gradle 5.6.2 + +commit 4407f3af1b485826b088c0c9af56ae6ebf3de05d +Author: Jim Ferenczi +Date: Thu Sep 12 13:46:23 2019 +0200 + + Delay the creation of SubSearchContext to the FetchSubPhase (#46598) + + This change delays the creation of the SubSearchContext for nested and parent/child inner_hits + to the fetch sub phase in order to ensure that a SearchContext can built entirely from a + QueryShardContext. This commit also adds a validation step to the inner hits builder that ensures that we fail the request early if the inner hits path is invalid. + + Relates #46523 + +commit 60db630abda8adf26ebb70ce0a86f783874dc638 +Author: James Rodewig +Date: Thu Sep 12 08:47:22 2019 -0400 + + [DOCS] Add missing mention of current version to snapshot docs (#46516) (#46658) + +commit 5c85b0998b46154b40ab55ea8790f8a7a16fd9a7 +Author: David Turner +Date: Thu Sep 12 11:07:34 2019 +0100 + + Clarify that discovery ignores master-ineligibles (#44835) + + The changes in #32006 mean that the discovery process can no longer use + master-ineligible nodes as a stepping-stone between master-eligible nodes. + This was normally an indication of a strange and possibly-fragile configuration + and was not recommended, but this commit adds a note to the breaking changes + docs to note that this kind of configuration is more obviously broken in recent + versions. + +commit 9f2c5632fe42c62a87733f228dcd336b494f2886 +Author: Alpar Torok +Date: Thu Sep 12 11:31:08 2019 +0300 + + Add support for tests.jvm.argline in testclusters (#46540) + + We have CI jobs that make use of this for testing alternative garbadge + collectors. + +commit e57756492a79922e88938e0ef822349f149c08d3 +Author: Luca Cavanna +Date: Wed Sep 11 13:42:32 2019 +0200 + + Update http-core and http-client dependencies (#46549) + + Relates to #45808 + Closes #45577 + +commit de47ea2cf4efe7e4c88a0fbef9f597ffcc0f3f18 +Author: Luca Cavanna +Date: Wed Sep 11 13:53:57 2019 +0200 + + Add SearchRestCancellationIT + + This test verifies automatic cancellation of search requests on connection close. + It was previously not present in 7.x as the http client was subject do a bug which + made testing cancellation of requests impossible. Now that the bug is fixed upstream, + we can also backport this test + +commit 030d43a76a5b70a3a19dca0da2c9462b8a8ca9b5 +Author: Jilles van Gurp +Date: Wed Aug 28 10:59:33 2019 +0200 + + Add a few notes on Cancellable to the LLRC and HLRC docs. (#45912) + + Add a section to both the low level and high level client documentation on asynchronous usage and `Cancellable` added for #44802 + + Co-Authored-By: Lee Hinman + +commit 60f40d76385ee8e052e82f652b75825e227b66b4 +Author: Jilles van Gurp +Date: Fri Aug 23 16:38:15 2019 +0200 + + Expose the ability to cancel async requests in REST high-level client (#45688) + + This commits makes all the async methods in the high level client return the `Cancellable` object that the low level client now exposes. + + Relates to #45379 + Closes #44802 + +commit cfb186afaf2bd82cf24efcecb01b65283e95f426 +Author: Luca Cavanna +Date: Tue Aug 13 16:48:06 2019 +0200 + + Add support for cancelling async requests in low-level REST client (#45379) + + The low-level REST client exposes a `performRequestAsync` method that + allows to send async requests, but today it does not expose the ability + to cancel such requests. That is something that the underlying apache + async http client supports, and it makes sense for us to expose. + + This commit adds a return value to the `performRequestAsync` method, + which is backwards compatible. A `Cancellable` object gets returned, + which exposes a `cancel` public method. When calling `cancel`, the + on-going request associated with the returned `Cancellable` instance + will be cancelled by calling its `abort` method. This works throughout + multiple retries, though some special care was needed for the case where + `cancel` is called between different attempts (when one attempt has + failed and the consecutive one has not been sent yet). + + Note that cancelling a request on the client side does not automatically + translate to cancelling the server side execution of it. That needs to be + specifically implemented, which is on the work for the search API (see #43332). + + Relates to #44802 + +commit 1c706656c2495eddf3642f3533adeca4811beab4 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Sep 12 09:11:50 2019 +0200 + + Wait for events in translog stats tests (#46586) + + When waiting for no initializing shards we also have to wait for events + when we have more than one node in the cluster. When the primary is + started, there is a short period of time, where neither the primary nor + any of the replicas are initializing. + + Closes #46535 + +commit ec5592ed76b56bd80a53c6e73fdbf9b1e58522b6 +Author: Lisa Cawley +Date: Wed Sep 11 16:32:47 2019 -0700 + + [DOCS] Adds missing icons to Watcher HLRC APIs (#46626) + +commit 4e371a220b9de91a83a0a3101fa33628870428eb +Author: Lisa Cawley +Date: Wed Sep 11 15:45:07 2019 -0700 + + [DOCS] Adds missing icons to ILM HLRC APIs (#46633) + +commit c3b6e6fe9684e987741ce0195d121c3778bea25b +Author: Lisa Cawley +Date: Wed Sep 11 15:35:42 2019 -0700 + + [DOCS] Adds missing icons to CCR HLRC APIs (#46631) + +commit 8200010e0ac7018a8f85302231833e6d2004b137 +Author: Lisa Cawley +Date: Wed Sep 11 15:20:36 2019 -0700 + + [DOCS] Adds missing icons to Graph HLRC APIs (#46630) + +commit 769f42bdd7d7c4612d6f00906928c5bf79a542c9 +Author: Lisa Cawley +Date: Wed Sep 11 13:19:13 2019 -0700 + + [DOCS] Add missing icons to security HLRC APIs (#46619) + +commit 35cb93248d5ac308dd5454be0d222a360bf456a0 +Author: Igor Motov +Date: Wed Sep 11 14:53:10 2019 -0400 + + Geo: fix indexing of west to east linestrings crossing the antimeridian (#46601) + + Fixes that way linestrings that are crossing the antimeridian are + indexed due to a normalization bug these lines were decomposed into + a line segment that was stretching entire globe. + + Fixes #43775 + +commit 6dc8ed5d573cf705d8664025c63edd9306f30607 +Author: Zachary Tong +Date: Wed Sep 11 17:00:28 2019 -0400 + + [7.x Backport] Refactor AllocatedPersistentTask#init(), move rollup ctor logic (#46406) + + This makes the AllocatedPersistentTask#init() method protected so that + implementing classes can perform their initialization logic there, + instead of the constructor. Rollup's task is adjusted to use this + init method. + + It also slightly refactors the methods to se a static logger in the + AllocatedTask instead of passing it in via an argument. This is + simpler, logged messages come from the task instead of the + service, and is easier for tests + +commit ffeeb410668cd9ddfa2c69096fc530b6568e050b +Author: Jason Tedor +Date: Wed Sep 11 16:30:43 2019 -0400 + + Fix muting of translog stats on closed indices test + + We do not allow multiple skip sections, so this commit comments out the + existing skip section. + + Relates #46535 + +commit 27da0ea603eb54aa7647e79b57862c2eac7ed16e +Author: Alpar Torok +Date: Tue Sep 10 14:09:09 2019 +0300 + + Mute translog stats on closed indices REST test + + This commit mutes a test of the translog stats on a closed indices with + soft-deletes enabled test. This test failure is being tracked in #46535. + +commit f9bf10f2b67170cb9c76e35777c90002b586ec7c +Author: James Rodewig +Date: Wed Sep 11 15:55:57 2019 -0400 + + [DOCS] Change "a SSL" to "an SSL" in the Java docs (#46524) (#46618) + +commit 941c215d52a04a95a7bdecd800125173cd57f40a +Author: Jason Tedor +Date: Wed Sep 11 15:20:08 2019 -0400 + + Favor java.vendor.version over java.vendor (#46615) + + In some cases (for example some AdoptOpenJDK builds), the java.vendor is + mistakenly populated as "Oracle Corporation" while the real value is + under "java.vendor.version". Since "java.vendor.version" is mandatory + since JDK 10, this commit changes to use "java.vendor.version" as the + favored system property to find the JVM vendor, and we fallback to + "java.vendor" if this is not populated (as happens in some Oracle + builds). Ugh. + +commit e609011754ec145e17d8a8a4d12f1cf0aeb3c240 +Author: Lisa Cawley +Date: Wed Sep 11 11:48:32 2019 -0700 + + [DOCS] Add missing icons to rollup HLRC APIs (#46617) + +commit d956509394a848f92764e219297e00758b280f50 +Author: Marios Trivyzas +Date: Wed Sep 11 21:09:40 2019 +0300 + + SQL: Implement DATE_TRUNC function (#46473) + + DATE_TRUNC(, ) is a function that allows + the user to truncate a timestamp to the specified field by zeroing out + the rest of the fields. The function is implemented according to the + spec from PostgreSQL: https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC + + Closes: #46319 + (cherry picked from commit b37e96712db1aace09f17b574eb02ff6b942a297) + +commit fa54de7789e3b3595531dd34ff66879570116829 +Author: Lisa Cawley +Date: Wed Sep 11 11:21:45 2019 -0700 + + [DOCS] Add missing icons to transform HLRC APIs (#46616) + +commit 86290cb3d92e45f4f7cc657e04e11bea8717e9ef +Author: Ryan Ernst +Date: Wed Sep 11 10:55:31 2019 -0700 + + Make reuse of sql test code explicit (#45884) + + The sql project uses a common set of security tests, which are run in + subprojects. Currently these are shared through a shared directory, but + this is not setup correctly to ensure it is built before tests run. This + commit changes the test classes to be an artifact of the sql/qa/security + project and makes the test runner use the built artifact (a directory of + classes) for tests. + + closes #45866 + +commit 35bf92cdac332fa2568080d4082e59c41b3a00b9 +Author: James Rodewig +Date: Wed Sep 11 13:52:20 2019 -0400 + + [DOCS] Reformat enrich stats API (#46600) + +commit fa9327cdb9b0776622526d07b1a2a1e03e36c8a2 +Author: Ryan Ernst +Date: Wed Sep 11 09:53:52 2019 -0700 + + Add more meaningful keystore version mismatch errors (#46291) + + This commit changes the version bounds of keystore reading to give + better error messages when a user has a too new or too old format. + + relates #44624 + +commit 44c441240622ef78027732e14c03cc6cb93b6424 +Author: Thibault Richard +Date: Wed Sep 11 18:53:57 2019 +0200 + + [DOCS] Fix backquote in the list of realm types (#46530) + +commit 09a9cefaa0cc5e7dae7ff77a33a79ad9d9c8b79c +Author: Lee Hinman +Date: Wed Sep 11 10:28:48 2019 -0600 + + Handle partial failure retrieving segments in SegmentCountStep (#46556) + + Since the `IndicesSegmentsRequest` scatters to all shards for the index, + it's possible that some of the shards may fail. This adds failure + handling and logging (since this is a best-effort step in the first + place) for this case. + +commit 0963e78164c46eb625a35a480f5bb8b3b00e86fa +Author: Marios Trivyzas +Date: Wed Sep 11 18:27:04 2019 +0300 + + SQL: Fix issue with common type resolution (#46565) + + Many scalar functions try to find out the common type between their + arguments in order to set it as their return time, e.g.: + for `float + double` the common type which is set as the return type + of the + operation is `double`. + + Previously, for data types TEXT and KEYWORD (string data types) there + was no common data type found and null was returned causing NPEs when + the function was trying to resolve the return data type. + + Fixes: #46551 + (cherry picked from commit 291017d69dfc810707c3c7c692f5a50af431b790) + +commit 043471c643d1311543c27a8b016aadf932d659bb +Author: James Rodewig +Date: Wed Sep 11 12:06:42 2019 -0400 + + [DOCS] Minor improvement to the nested aggregation docs (#46475) (#46604) + + * Minor improvement to the nested aggregation docs + + * The attributes name and resellers.name were rather confusing, + especially since the first one was dynamically mapped and not shown + in the documentation (you had to read the test to see it). This + change introduces a unique name for the nested attribute and adds + the example document to the documentation. + * Change the index name from "index" to something more speaking. + + * Update docs/reference/aggregations/bucket/nested-aggregation.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/reference/aggregations/bucket/nested-aggregation.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/reference/aggregations/bucket/nested-aggregation.asciidoc + + Co-Authored-By: James Rodewig + +commit dcdfc382c77043d405de790e1a3aa328fed2a701 +Author: Lee Hinman +Date: Wed Sep 11 09:54:02 2019 -0600 + + Fix ILM start and stop doc tests (#46559) + + This uses whatever the server retrieves, rather than hardcoded + "STOPPING" and "STOPPED" since the server may go to STOPPED before the + request is issued. + + Resolves #46528 + +commit 52d7b03b496e61f780f25aa82411d952fb56d95d +Author: Lee Hinman +Date: Wed Sep 11 09:32:16 2019 -0600 + + Wait for no snapshots in state in testRetentionWhileSnapshotIn… (#46573) + + This commit adds a wait/check for all running snapshots to be cleared + before taking another snapshot. The previous snapshot was successful but + had not yet been cleared from the cluster state, so the second snapshot + failed due to a `ConcurrentSnapshotException`. + + Resolves #46508 + +commit c0ec6ade4bab27f55f64e9722cd5cfce4d31be23 +Author: Lisa Cawley +Date: Wed Sep 11 08:44:03 2019 -0700 + + [DOCS] Adds transform content (#46575) (#46578) + +commit 461de5b58e6b165a35e86acec606d793bb3ad300 +Author: David Roberts +Date: Wed Sep 11 16:29:30 2019 +0100 + + [TEST] Remove incorrect data frame analytics state assertion (#46597) + + After starting the analytics job and checking its state + the state can be any of "started", "reindexing" or + "analyzing" depending on how quickly the work is done. + +commit dea472c6fb1f235656dc54279007e03a65ea7816 +Author: Lisa Cawley +Date: Wed Sep 11 08:26:25 2019 -0700 + + [DOCS] Adds machine learning PRs to release notes (#46564) + +commit 07a014026030d5fb0f6062448cdea516cdb60781 +Author: David Roberts +Date: Wed Sep 11 16:27:26 2019 +0100 + + [ML-DataFrame] Ensure latest index template exists before indexing docs (#46595) + + When upgrading data nodes to a newer version before + master nodes there was a risk that a transform running + on an upgraded data node would index a document into + the new transforms internal index before its index + template was created. This would cause the index to + be created with entirely dynamic mappings. + + This change introduces a check before indexing any + internal transforms document to ensure that the required + index template exists and create it if it doesn't. + + Backport of #46553 + +commit ccf656a9d065ab979febd4cb0c249cf8d48cc517 +Author: Mark Vieira +Date: Wed Sep 11 08:24:32 2019 -0700 + + Repository plugin test cacheability fixes (#46572) + +commit 23bf310c849c77e50ab87dbb59ea42d9412fe187 +Author: Jim Ferenczi +Date: Wed Sep 11 16:12:34 2019 +0200 + + Replace the SearchContext with QueryShardContext when building aggregator factories (#46527) + + This commit replaces the `SearchContext` with the `QueryShardContext` when building aggregator factories. Aggregator factories are part of the `SearchContext` so they shouldn't require a `SearchContext` to create them. + The main changes here are the signatures of `AggregationBuilder#build` that now takes a `QueryShardContext` and `AggregatorFactory#createInternal` that passes the `SearchContext` to build the `Aggregator`. + + Relates #46523 + +commit 27c15f137edbae7696f2ea749e1e339a61266c4e +Author: Armin Braun +Date: Wed Sep 11 16:34:24 2019 +0200 + + Remove Unused Method from BlobStoreRepository (#46204) (#46593) + + This method isn't used anymore and I forgot to delete it. + +commit 8c9f15db443d0e358050fdc5b31427386b7061d4 +Author: William Brafford +Date: Wed Sep 11 09:33:00 2019 -0400 + + Fix Path comparisons for Windows tests (#46503) (#46566) + + * Fix Path comparisons for Windows tests + + The test NodeEnvironmentTests#testCustonDataPaths worked just fine on + Darwin and Linux, but the comparison was breaking in Windows because one + path had the "C:\" prefix and the other one didn't. The simple fix is to + compare absolute paths rather than potentially relative ones. + +commit aa0c586b730b050746ec4bb96a8eaedaddb1a2c4 +Author: Christoph Büscher +Date: Wed Sep 11 14:55:30 2019 +0200 + + Deprecate `_field_names` disabling (#42854) + + Currently we allow `_field_names` fields to be disabled explicitely, but since + the overhead is negligible now we decided to keep it turned on by default and + deprecate the `enable` option on the field type. This change adds a deprecation + warning whenever this setting is used, going forward we want to ignore and finally + remove it. + + Closes #27239 + +commit efea581dcc8b60e4bc51e70f9692df1e889b705f +Author: Hendrik Muhs +Date: Wed Sep 11 14:50:08 2019 +0200 + + [7.x][Transform]Rename data frame plugin to transform: plugin and package names (#46583) + + rename data frame transform plugin to transform: + + - rename plugin data-frame to transform + - change all package names from o.e.*.dataframe.* to o.e.*.transform.* + - necessary changes to fix loading/testing + +commit 60ad099178e3756570f34158b53fb4c8c3cf4d7c +Author: Martijn van Groningen +Date: Wed Sep 11 14:26:42 2019 +0200 + + Add HLRC support for enrich get policy API. (#45970) + + Changed the signature of AbstractResponseTestCase#createServerTestInstance(...) + to include the randomly selected xcontent type. This is needed for the + creating a server response instance with a query which is represented as BytesReference. + Maybe this should go into a different change? + + This PR also includes HLRC docs for the get policy api. + + Relates to #32789 + +commit 41633cb9b50491528d2f0776f4c06d39df4d9594 +Author: Armin Braun +Date: Wed Sep 11 13:59:20 2019 +0200 + + More Efficient Ordering of Shard Upload Execution (#42791) (#46588) + + * More Efficient Ordering of Shard Upload Execution (#42791) + * Change the upload order of of snapshots to work file by file in parallel on the snapshot pool instead of merely shard-by-shard + * Inspired by #39657 + * Cleanup BlobStoreRepository Abort and Failure Handling (#46208) + +commit a4b0f669195ed8ef9a8e400cf1e6173aef92063b +Author: Martijn van Groningen +Date: Wed Sep 11 12:58:46 2019 +0200 + + Add enrich stats api (#46462) + + The enrich api returns enrich coordinator stats and + information about currently executing enrich policies. + + The coordinator stats include per ingest node: + * The current number of search requests in the queue. + * The total number of outstanding remote requests that + have been executed since node startup. Each remote + request is likely to include multiple search requests. + This depends on how much search requests are in the + queue at the time when the remote request is performed. + * The number of current outstanding remote requests. + * The total number of search requests that `enrich` + processors have executed since node startup. + + The current execution policies stats include: + * The name of policy that is executing + * A full blow task info object that is executing the policy. + + Relates to #32789 + +commit b5dbb3f2b33a9eb74e2bb587d6ee55e216c1a601 +Merge: c79a8e448d5 a27d075db43 +Author: Martijn van Groningen +Date: Wed Sep 11 13:30:52 2019 +0200 + + Merge branch 'enrich_backport_docs_commits' into enrich-7.x + +commit a27d075db4366a0b1204c308a99c62d442db259f +Author: James Rodewig +Date: Mon Sep 9 08:44:56 2019 -0400 + + [DOCS] Update "Enrich your data" tutorials (#46417) + + * Move enrich docs to separate file + + * Rewrite enrich processor tutorial + +commit d74d9953824ed4ae2c1b6c9a687c2eba5bb3cbee +Author: James Rodewig +Date: Wed Sep 4 14:11:52 2019 -0400 + + [DOCS] Separate Enrich API Docs (#46286) + + * Add enrich policy common parameter + + * Add enrich APIs to REST APIs index + + * Add put enrich policy API docs + + * Add get enrich policy API docs + + * Add delete enrich policy API docs + + * Add execute enrich policy API docs + +commit 80bb08fbda12217d9d17d2c8d240cdceb8b1dbd3 +Author: Jim Ferenczi +Date: Wed Sep 11 12:00:32 2019 +0200 + + Replace the SearchContext with QueryShardContext when building collapsing context (#46543) + + This commit replaces the `SearchContext` with the `QueryShardContext` when building collapsing conteext + Collapse context is part of the `SearchContext` so it shouldn't require a `SearchContext` to create one. + + Relates #46523 + +commit 425b1a77e84a6267ba5471e3164b4e7ab0ef7cf1 +Author: Jim Ferenczi +Date: Wed Sep 11 11:55:41 2019 +0200 + + Add more context to QueryShardContext (#46584) + + This change adds an IndexSearcher and the node's BigArrays in the QueryShardContext. + It's a spin off of #46527 as this change is required to allow aggregation builder to solely use the + query shard context. + + Relates #46523 + +commit 579af626f53228ed62c0658fa9bb825cdc1fab6b +Author: Dimitris Athanasiou +Date: Wed Sep 11 13:18:42 2019 +0300 + + [7.x][ML] No error when datafeed stops during updating to started (#46495) (#46542) + + Investigating the test failure reported in #45518 it appears that + the datafeed task was not found during a tast state update. There + are only two places where such an update is performed: when we set + the state to `started` and when we set it to `stopping`. We handle + `ResourceNotFoundException` in the latter but not in the former. + + Thus the test reveals a rare race condition where the datafeed gets + requested to stop before we managed to update its state to `started`. + I could not reproduce this scenario but it would be my best guess. + + This commit catches `ResourceNotFoundException` while updating the + state to `started` and lets the task terminate smoothly. + + Closes #45518 + + Backport of #46495 + +commit e38e631dac67309276b99ef517d4f65dafa73249 +Author: Przemysław Witek +Date: Wed Sep 11 12:17:26 2019 +0200 + + [7.x] Implement DataFrameAnalyticsAuditMessage and DataFrameAnalyticsAuditor (#45967) (#46519) + +commit 35810bd2aefd1bce65ccb55cdd282ae44fc028c9 +Author: Ioannis Kakavas +Date: Wed Sep 11 13:13:59 2019 +0300 + + Enforce realm name uniqueness (#46580) + + We depend on file realms being unique in a number of places. Pre + 7.0 this was enforced by the fact that the multiple realm types + with different name would mean identical configuration keys and + cause configuration parsing errors. Since we intoduced affix + settings for realms this is not the case any more as the realm type + is part of the configuration key. + This change adds a check when building realms which will explicitly + fail if multiple realms are defined with the same name. + + Backport of #46253 + +commit f8d5145472411160dfa8b26d7bcd33f1598edc16 +Author: Armin Braun +Date: Wed Sep 11 11:58:42 2019 +0200 + + Fix SnapshotStatusApisIT (#46563) (#46582) + + Obviously we have to run the status request again to busy wait for + the `STARTED` state, just busy waiting on an existing response + won't do anything. + + Closes #45917 + +commit 31697c71bd2b2354f988eda54efb1280d36faccf +Author: Colin Goodheart-Smithe +Date: Wed Sep 11 10:51:04 2019 +0100 + + Updates 7.4.0 release notes + +commit c79a8e448d573302ad86fd8e273a6216308e6a61 +Author: Martijn van Groningen +Date: Wed Sep 11 11:39:58 2019 +0200 + + Convert enrich qa modules to use testclusters. + +commit ac52588a7a51a5cd8690c4ca44bf2d97a2b73a1d +Author: Colin Goodheart-Smithe +Date: Wed Sep 11 10:13:44 2019 +0100 + + Updates 7.4.0 release notes + +commit 8a48ef2a067373d5371952fd36a112fa7b14256e +Author: Martijn van Groningen +Date: Wed Sep 11 09:52:09 2019 +0200 + + fixed typo + +commit ef33a99e6eaa68949523a46aa551767087ccc1ed +Author: Martijn van Groningen +Date: Wed Sep 11 09:20:10 2019 +0200 + + Disable default features that are not needed for enrich indices. (#46525) + + Relates to #32789 + +commit 80064652f810a6bf9e2f13d2cdb88376506a3ab7 +Author: Tim Vernum +Date: Wed Sep 11 14:33:17 2019 +1000 + + Fallback to realm authc if ApiKey fails (#46552) + + This changes API-Key authentication to always fallback to the realm + chain if the API key is not valid. The previous behaviour was + inconsistent and would terminate on some failures, but continue to the + realm chain for others. + + Backport of: #46538 + +commit 7a2878b29bf869417e3f3dd174b9cda3aa4b2f74 +Author: Gordon Brown +Date: Tue Sep 10 12:41:36 2019 -0600 + + Fix class used to initialize logger in Watcher (#46467) + + This class has been using a logger configured for a different class for + quite a while. While the circumstance in which it logs is rare, it + should still use the correct logger. + +commit 10da998dfaa5ea2cc054267629a22564172225c9 +Author: Julie Tibshirani +Date: Tue Sep 10 11:01:45 2019 -0700 + + Expand documentation around global ordinals. (#46517) + + This commit updates the eager_global_ordinals documentation to give more + background on what global ordinals are and when they are used. The docs also now + mention that global ordinal loading may be expensive, and describes the cases + where in which loading them can be avoided. + +commit 20e93bca6b695d7114f0b484a6d1c5575c8e317c +Author: Abhilash Bolla <2282894+ivssh@users.noreply.github.com> +Date: Tue Sep 10 22:16:06 2019 +0530 + + Fixed grammar in pattern replace char filter docs. (#46546) + + Minor grammar fix in the pattern replace char filter docs. + +commit 1b9bd9a0a84b8e2bdf09b938b79a08af15074c4d +Author: Julie Tibshirani +Date: Mon Sep 9 11:10:54 2019 -0700 + + Use a literal block in the field data docs. (#46469) + + Currently we use `quote`, which renders a bit strangely on the website. + +commit 70c00621dbba3f195050a03169842d06e94c02a5 +Author: Lisa Cawley +Date: Tue Sep 10 10:32:51 2019 -0700 + + [DOCS] Add missing xpack role attributes (#46468) + +commit 7f9e2f4d91abfb60858cc768c33fdc4248571f52 +Author: Nhat Nguyen +Date: Mon Sep 9 09:38:49 2019 -0400 + + Ensure no ongoing peer recovery in translog yaml test (#46476) + + We leave replicas unassigned until we reroute after the primary shard + starts. If a cluster health request with wait_for_no_initializing_shards + is executed before the reroute, it will return immediately although + there will be some initializing replicas. Peer recoveries of those + shards can prevent translog on the primary from trimming. + + We add wait_for_events to the cluster health request so that it will + execute after the reroute. + + Closes #46425 + +commit 972d3569c1ebce1b28116986fef97f5e8b3acddd +Author: Mark Vieira +Date: Tue Sep 10 08:32:36 2019 -0700 + + Disable local build cache in CI (#46505) + + + (cherry picked from commit 4dcc226e142057ec44b313edf883e1c5401ec610) + +commit e5d315f6e1549fa53768839a483ee314fabbdf72 +Author: Mark Vieira +Date: Tue Sep 10 08:24:05 2019 -0700 + + Ensure rest api specs are always copied before using test classpath (#46514) + + (cherry picked from commit 45202903b4fea3a43f62594fd357ab3c98c3dd15) + +commit 7461259ba6c3090250b0cfe674a21d7e90274059 +Author: Lisa Cawley +Date: Tue Sep 10 08:26:56 2019 -0700 + + [DOCS] Adds missing icons to ML HLRC APIs (#46515) + +commit cdc3a260af5306f9a964819eb556a3b7c04c2b7a +Author: Lee Hinman +Date: Tue Sep 10 09:08:09 2019 -0600 + + Add retention to Snapshot Lifecycle Management (backport of #4… (#46506) + + * Add retention to Snapshot Lifecycle Management (#46407) + + This commit adds retention to the existing Snapshot Lifecycle Management feature (#38461) as described in #43663. This allows a user to configure SLM to automatically delete older snapshots based on a number of criteria. + + An example policy would look like: + + ``` + PUT /_slm/policy/snapshot-every-day + { + "schedule": "0 30 2 * * ?", + "name": "", + "repository": "my-s3-repository", + "config": { + "indices": ["foo-*", "important"] + }, + // Newly configured retention options + "retention": { + // Snapshots should be deleted after 14 days + "expire_after": "14d", + // Keep a maximum of thirty snapshots + "max_count": 30, + // Keep a minimum of the four most recent snapshots + "min_count": 4 + } + } + ``` + + SLM Retention is run on a scheduled configurable with the `slm.retention_schedule` setting, which supports cron expressions. Deletions are run for a configurable time bounded by the `slm.retention_duration` setting, which defaults to 1 hour. + + Included in this work is a new SLM stats API endpoint available through + + ``` json + GET /_slm/stats + ``` + + That returns statistics about snapshot taken and deleted, as well as successful retention runs, failures, and the time spent deleting snapshots. #45362 has more information as well as an example of the output. These stats are also included when retrieving SLM policies via the API. + + * Add base framework for snapshot retention (#43605) + + * Add base framework for snapshot retention + + This adds a basic `SnapshotRetentionService` and `SnapshotRetentionTask` + to start as the basis for SLM's retention implementation. + + Relates to #38461 + + * Remove extraneous 'public' + + * Use a local var instead of reading class var repeatedly + + * Add SnapshotRetentionConfiguration for retention configuration (#43777) + + * Add SnapshotRetentionConfiguration for retention configuration + + This commit adds the `SnapshotRetentionConfiguration` class and its HLRC + counterpart to encapsulate the configuration for SLM retention. + Currently only a single parameter is supported as an example (we still + need to discuss the different options we want to support and their + names) to keep the size of the PR down. It also does not yet include version serialization checks + since the original SLM branch has not yet been merged. + + Relates to #43663 + + * Fix REST tests + + * Fix more documentation + + * Use Objects.equals to avoid NPE + + * Put `randomSnapshotLifecyclePolicy` in only one place + + * Occasionally return retention with no configuration + + * Implement SnapshotRetentionTask's snapshot filtering and delet… (#44764) + + * Implement SnapshotRetentionTask's snapshot filtering and deletion + + This commit implements the snapshot filtering and deletion for + `SnapshotRetentionTask`. Currently only the expire-after age is used for + determining whether a snapshot is eligible for deletion. + + Relates to #43663 + + * Fix deletes running on the wrong thread + + * Handle missing or null policy in snap metadata differently + + * Convert Tuple> to Map> + + * Use the `OriginSettingClient` to work with security, enhance logging + + * Prevent NPE in test by mocking Client + + * Allow empty/missing SLM retention configuration (#45018) + + Semi-related to #44465, this allows the `"retention"` configuration map + to be missing. + + Relates to #43663 + + * Add min_count and max_count as SLM retention predicates (#44926) + + This adds the configuration options for `min_count` and `max_count` as + well as the logic for determining whether a snapshot meets this criteria + to SLM's retention feature. + + These options are optional and one, two, or all three can be specified + in an SLM policy. + + Relates to #43663 + + * Time-bound deletion of snapshots in retention delete function (#45065) + + * Time-bound deletion of snapshots in retention delete function + + With a cluster that has a large number of snapshots, it's possible that + snapshot deletion can take a very long time (especially since deletes + currently have to happen in a serial fashion). To prevent snapshot + deletion from taking forever in a cluster and blocking other operations, + this commit adds a setting to allow configuring a maximum time to spend + deletion snapshots during retention. This dynamic setting defaults to 1 + hour and is best-effort, meaning that it doesn't hard stop a deletion + at an hour mark, but ensures that once the time has passed, all + subsequent deletions are deferred until the next retention cycle. + + Relates to #43663 + + * Wow snapshots suuuure can take a long time. + + * Use a LongSupplier instead of actually sleeping + + * Remove TestLogging annotation + + * Remove rate limiting + + * Add SLM metrics gathering and endpoint (#45362) + + * Add SLM metrics gathering and endpoint + + This commit adds the infrastructure to gather metrics about the different SLM actions that a cluster + takes. These actions are stored in `SnapshotLifecycleStats` and perpetuated in cluster state. The + stats stored include the number of snapshots taken, failed, deleted, the number of retention runs, + as well as per-policy counts for snapshots taken, failed, and deleted. It also includes the amount + of time spent deleting snapshots from SLM retention. + + This commit also adds an endpoint for retrieving all stats (further commits will expose this in the + SLM get-policy API) that looks like: + + ``` + GET /_slm/stats + { + "retention_runs" : 13, + "retention_failed" : 0, + "retention_timed_out" : 0, + "retention_deletion_time" : "1.4s", + "retention_deletion_time_millis" : 1404, + "policy_metrics" : { + "daily-snapshots2" : { + "snapshots_taken" : 7, + "snapshots_failed" : 0, + "snapshots_deleted" : 6, + "snapshot_deletion_failures" : 0 + }, + "daily-snapshots" : { + "snapshots_taken" : 12, + "snapshots_failed" : 0, + "snapshots_deleted" : 12, + "snapshot_deletion_failures" : 6 + } + }, + "total_snapshots_taken" : 19, + "total_snapshots_failed" : 0, + "total_snapshots_deleted" : 18, + "total_snapshot_deletion_failures" : 6 + } + ``` + + This does not yet include HLRC for this, as this commit is quite large on its own. That will be + added in a subsequent commit. + + Relates to #43663 + + * Version qualify serialization + + * Initialize counters outside constructor + + * Use computeIfAbsent instead of being too verbose + + * Move part of XContent generation into subclass + + * Fix REST action for master merge + + * Unused import + + * Record history of SLM retention actions (#45513) + + This commit records the deletion of snapshots by the retention component + of SLM into the SLM history index for the purposes of reviewing operations + taken by SLM and alerting. + + * Retry SLM retention after currently running snapshot completes (#45802) + + * Retry SLM retention after currently running snapshot completes + + This commit adds a ClusterStateObserver to wait until the currently + running snapshot is complete before proceeding with snapshot deletion. + SLM retention waits for the maximum allowed deletion time for the + snapshot to complete, however, the waiting time is not factored into + the limit on actual deletions. + + Relates to #43663 + + * Increase timeout waiting for snapshot completion + + * Apply patch + + From https://github.com/original-brownbear/elasticsearch/commit/2374316f0d1912c9e1498bece195546a1dc60bce.patch + + * Rename test variables + + * [TEST] Be less strict for stats checking + + * Skip SLM retention if ILM is STOPPING or STOPPED (#45869) + + This adds a check to ensure we take no action during SLM retention if + ILM is currently stopped or in the process of stopping. + + Relates to #43663 + + * Check all actions preventing snapshot delete during retention (#45992) + + * Check all actions preventing snapshot delete during retention run + + Previously we only checked to see if a snapshot was currently running, + but it turns out that more things can block snapshot deletion. This + changes the check to be a check for: + + - a snapshot currently running + - a deletion already in progress + - a repo cleanup in progress + - a restore currently running + + This was found by CI where a third party delete in a test caused SLM + retention deletion to throw an exception. + + Relates to #43663 + + * Add unit test for okayToDeleteSnapshots + + * Fix bug where SLM retention task would be scheduled on every node + + * Enhance test logging + + * Ignore if snapshot is already deleted + + * Missing import + + * Fix SnapshotRetentionServiceTests + + * Expose SLM policy stats in get SLM policy API (#45989) + + This also adds support for the SLM stats endpoint to the high level rest client. + + Retrieving a policy now looks like: + + ```json + { + "daily-snapshots" : { + "version": 1, + "modified_date": "2019-04-23T01:30:00.000Z", + "modified_date_millis": 1556048137314, + "policy" : { + "schedule": "0 30 1 * * ?", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"], + "ignore_unavailable": false, + "include_global_state": false + }, + "retention": {} + }, + "stats": { + "snapshots_taken": 0, + "snapshots_failed": 0, + "snapshots_deleted": 0, + "snapshot_deletion_failures": 0 + }, + "next_execution": "2019-04-24T01:30:00.000Z", + "next_execution_millis": 1556048160000 + } + } + ``` + + Relates to #43663 + + * Rewrite SnapshotLifecycleIT as as ESIntegTestCase (#46356) + + * Rewrite SnapshotLifecycleIT as as ESIntegTestCase + + This commit splits `SnapshotLifecycleIT` into two different tests. + `SnapshotLifecycleRestIT` which includes the tests that do not require + slow repositories, and `SLMSnapshotBlockingIntegTests` which is now an + integration test using `MockRepository` to simulate a snapshot being in + progress. + + Relates to #43663 + Resolves #46205 + + * Add error logging when exceptions are thrown + + * Update serialization versions + + * Fix type inference + + * Use non-Cancellable HLRC return value + + * Fix Client mocking in test + + * Fix SLMSnapshotBlockingIntegTests for 7.x branch + + * Update SnapshotRetentionTask for non-multi-repo snapshot retrieval + + * Add serialization guards for SnapshotLifecyclePolicy + +commit 9304f5c8896370000737558a16452f0278980fb0 +Author: Michael Basnight +Date: Tue Sep 10 09:58:54 2019 -0500 + + Ensure enrich executes on master node only (#46448) + + The previous transport action was a read action, which under the right + set of circumstances can execute on a coordinating node. This commit + ensures that cannot happen. + +commit 2c5f9b558b42a8dc4a4045095961f7ce0f6a02c5 +Author: Mayya Sharipova +Date: Tue Sep 10 08:26:20 2019 -0400 + + Fix highlighting for script_score query (#46507) + +commit 690164d0beeee6082e73d87c1024d11c77bc69f0 +Author: Ioannis Kakavas +Date: Tue Sep 10 14:38:35 2019 +0300 + + Change EmailSslTest for FIPS 140 JVMs (#46278) + + This commit changes the SSLContext for the email server we use in + the tests so that it loads its key material from an in memory + keystore (that is in turn built from a pair of PEM encoded private key + and certificate) instead of a PKCS#12 one. This is done so that when + we run our tests in FIPS 140-2 JVMs, the keystore is of a type that the + Security Provider actually supports. + + This also mutes testCanSendMessageToSmtpServerByDisablingVerification + as we can't run tests with verification set to `none` in FIPS 140 + JVMs. + +commit 0ac52d0e7236a68a8dbc84ab2cb9e687c6c5dfeb +Author: Alpar Torok +Date: Tue Sep 10 13:27:37 2019 +0300 + + Mute test in 7.x + + Tracked in #46529 + +commit 6c67b539321f234ecacd9c6aae47812c27f02dae +Author: David Turner +Date: Tue Sep 10 11:15:10 2019 +0100 + + Load metadata at start time not construction time (#46326) + + Today we load the metadata from disk while constructing the node. However there + is no real need to do so, and this commit moves that code to run later while + the node is starting instead. + +commit b40ac6dee7f9a01cda40842ba5ea2b979415386d +Author: Alpar Torok +Date: Tue Sep 10 12:30:42 2019 +0300 + + mute on 7.x fo windows + + Tracking #44942 + +commit 7125c101e6484c84cd7aca0ff2a69181b37c429d +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Sep 10 08:46:37 2019 +0200 + + HLRC multisearchTemplate forgot params (#46492) + + Since 7.3, the request converter for multiSearchTemplate would silently + not set the two request parameters `typed_keys` and + `max_concurrent_searches`. + + Closes #46488 + +commit 9fce5a99d8a1c717317d967f307dfdd7fdad2eeb +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Sep 9 21:48:26 2019 +0200 + + Rest Controller wildcard registration (#46487) + + Registering two different http methods on the same path using different + wildcard names would result in the last wildcard name being active only. + Now throw an exception instead. + + Closes #46482 + +commit e21deae535033361d7c23cd4596a3e38ddeffefb +Author: Przemysław Witek +Date: Mon Sep 9 21:01:27 2019 +0200 + + Disallow persisting any documents when datafeed is isolated (#46485) (#46490) + +commit b59ecde041c2cc67d7f89798c843c8247489948b +Author: James Rodewig +Date: Mon Sep 9 13:38:14 2019 -0400 + + [DOCS] [2 of 5] Change // CONSOLE comments to [source,console] (#46353) (#46502) + +commit b21e4171814c99bc18a5dfb9273384a72f1559d8 +Author: Zachary Tong +Date: Mon Sep 9 13:35:08 2019 -0400 + + [DOCS] Add 7.3.2 Release Notes (#46454) + +commit e253ee6ba641ae24e1e505a589954124c68a9f4e +Author: James Rodewig +Date: Mon Sep 9 12:35:50 2019 -0400 + + [DOCS] Change // CONSOLE comments to [source,console] (#46440) (#46494) + +commit 8d17527050cd9700d6ea18b43a337c29a07a7eb9 +Author: Zachary Tong +Date: Mon Sep 9 10:01:36 2019 -0400 + + [TEST] create larger cuckoo filters for tests (#46457) + + The cuckoofilters could be randomly created with too small of + capacity or precision, which means that they can only absorb a few + values before collisions start to make all filters look identical. + + This increases the size of filters we generate (capacity >> than + the test cases) and lower fpp rate. + +commit 88bed09119618f5777a6889c4db93be4ad40a0d8 +Author: Tanguy Leroux +Date: Mon Sep 9 15:51:24 2019 +0200 + + Mutualize code in cloud-based repository integration tests (#46483) + + This commit factors out some common code between the cloud-based + repository integration tests that were recently improved. + + Relates #46376 + +commit 8428f8e6e86103c3c6dfd6898db0246b3c3709f5 +Author: David Turner +Date: Mon Sep 9 14:46:49 2019 +0100 + + Remove trailing comma from nodes lists (#46484) + + Today when the membership of the cluster changes we log messages that describe + the change like this: + + added {{node-1}{OPdaTIGmSxaEXXOyg3o96w}{127.0.0.1}{127.0.0.1:9301}{di},} + + The trailing comma suggests there is some missing string that might contain + extra information, but in fact it's an artefact of how these messages are + constructed. This commit removes the trailing comma from these lists. + +commit ee3396735cd03da4b68c0d2de0ed3936d1e30b32 +Author: Armin Braun +Date: Mon Sep 9 14:38:11 2019 +0200 + + Execute SnapshotsService Error Callback on Generic Thread (#46277) (#46480) + + I couldn't find a test for this, as it seems we only get + into this error handler on a bug. Regardless, we are + executing the snapshot finalization on the master update + thread here which shouldn't happen and will make + debugging a production issue resulting from this + trickier than it has to be (because we probably also + get a cluster state apply is slow warning in addition + to the original bug). + Used the generic pool here instead of the snapshot pool + because we're resolving the user callback here as + well and the generic pool seemed like the safer bet for + that. + +commit 31bee53fdda37563d1d00c0370c67c4ccda01019 +Author: Alpar Torok +Date: Mon Sep 9 15:19:37 2019 +0300 + + Fix the JVM we use for bwc nodes (#46314) + + Before this change we would run bwc nodes with their bundled jdk if + these supported it, so the passed in runtime JDK was not honored. + This became obvius when running with FIPS. + + Closes #41721 + +commit d5529cb0bb7b77d7a314731cf75eb575db982807 +Author: Suhel Khan +Date: Mon Sep 9 17:46:31 2019 +0530 + + [Docs] Fix typo in minimum-should-match.asciidoc (#46472) + +commit 0915bd7c6aade2da6570b6b232af283d2da8b428 +Author: Alexander Reelsen +Date: Mon Sep 9 13:41:20 2019 +0200 + + Update mustache dependency to 0.9.6 (#46243) + +commit 023cf44025edf08945c507ebd0692172699b8c3f +Author: Tanguy Leroux +Date: Mon Sep 9 09:59:01 2019 +0200 + + Inject random server errors in AzureBlobStoreRepositoryTests (#46371) + + This commit modifies the HTTP server used in + AzureBlobStoreRepositoryTests so that it randomly returns + server errors for any type of request executed by the Azure client. + +commit 8e3dc684544d2e4a398f6aa738556752004d3e7e +Author: Tanguy Leroux +Date: Mon Sep 9 09:57:43 2019 +0200 + + Inject random server errors in GoogleCloudStorageBlobStoreRepositoryTests (#46376) + + This commit modifies the HTTP server used in + GoogleCloudStorageBlobStoreRepositoryTests so that it randomly + returns server errors. The test does not inject server errors for the + following types of request: batch request, resumable upload request. + +commit cc092b1be1756c6452cd7d98bbce03d511780970 +Author: David Turner +Date: Mon Sep 9 07:54:02 2019 +0100 + + Add support for OneZoneInfrequentAccess storage (#46436) + + The `repository-s3` plugin has supported a storage class of `onezone_ia` since + the SDK upgrade in #30723, but we do not test or document this fact. This + commit adds this storage class to the docs and adds a test to ensure that the + documented storage classes are all accepted by S3 too. + + Fixes #30474 + +commit c057fce97846506f9519aa9919b9bfe5485b1b52 +Merge: ded98e50b79 24c3a1de3c6 +Author: Martijn van Groningen +Date: Mon Sep 9 08:40:54 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 24c3a1de3c621fdfaa467df5172c6bb54132a413 +Author: Nhat Nguyen +Date: Sat Sep 7 10:44:25 2019 -0400 + + Ignore replication for noop updates (#46458) + + Previously, we ignore replication for noop updates because they do not + have sequence numbers. Since #44603, we started assigning sequence + numbers to noop updates leading them to be replicated to replicas. + + This bug occurs only on 8.0 for it requires #41065 and #44603. + + Closes #46366 + +commit 7b26a8c041838da9c3160cbbff46ea39a90b1420 +Author: Andrei Stefan +Date: Sat Sep 7 09:14:02 2019 +0300 + + Use `null` schema response for `SYS TABLES` command. (#46386) + + (cherry picked from commit a6152f42a47a1ccd668e5892778c8bd2d3a78c4c) + +commit 7cf100ba071576c089605e6cc83f3466f4d32ef1 +Author: Andrei Stefan +Date: Sat Sep 7 09:13:42 2019 +0300 + + SQL: fix scripting for grouped by datetime functions (#46421) + + * Fix issue with painless scripting not being correctly generated when + datetime functions are used for GROUPing of an INTERVAL operation. + + (cherry picked from commit cb92828e8ec9d9d241bd6189e5835fd99f8b9a44) + +commit 1bb1c778858f6a371ff00e4c27409dd4e3f269a8 +Author: Armin Braun +Date: Sat Sep 7 07:40:16 2019 +0200 + + Increase REST-Test Client Timeout to 60s (#46455) (#46461) + + We are seeing requests take more than the default 30s + which leads to requests being retried and returning + unexpected failures like e.g. "index already exists" + because the initial requests that timed out, worked + out functionally anyway. + => double the timeout to reduce the likelihood of + the failures described in #46091 + => As suggested in the issue, we should in a follow-up + turn off retrying all-together probably + +commit a6cc0deaa06bc4cdfddfd7d3153053f658522e23 +Author: James Rodewig +Date: Fri Sep 6 16:46:41 2019 -0400 + + [DOCS] Remove cat request from Index Segments API requests (#46463) + +commit f04573f8e8954c174abf362d59cf2f025b2537b3 +Author: James Rodewig +Date: Fri Sep 6 16:09:09 2019 -0400 + + [DOCS] [5 of 5] Change // TESTRESPONSE comments to [source,console-results] (#46449) (#46459) + +commit 61756597ae3a4d0e3647e2abb6ab74459a79e608 +Author: James Rodewig +Date: Fri Sep 6 14:10:36 2019 -0400 + + [DOCS] Correct definition for `allow_no_indices` parameter (#46450) + +commit 6ae1ae9c9a0c28b5903496c19dbbc11a763a070e +Author: Anton <33043735+siffash@users.noreply.github.com> +Date: Fri Sep 6 19:01:34 2019 +0300 + + [Docs] Fix typo in field-names-field.asciidoc (#46430) + +commit b05c0f333da750b9666c6ab4b3c03cb758a817cd +Author: James Rodewig +Date: Fri Sep 6 11:35:27 2019 -0400 + + [DOCS] Add index alias definition to glossary (#46339) + +commit c46c57d4396e550e56d2a2a3cb23d1e4d549cc51 +Author: James Rodewig +Date: Fri Sep 6 11:31:13 2019 -0400 + + [DOCS] Change // CONSOLE comments to [source,console] (#46441) (#46451) + +commit 31b4e2f6dfdd4e389585e8bebfb66413aa5674cb +Author: James Rodewig +Date: Fri Sep 6 10:30:43 2019 -0400 + + [DOCS] Resort common-parms (#46419) (#46442) + +commit 7c7fb7e32de79ac2e240421337658a5f480adf2d +Author: David Roberts +Date: Fri Sep 6 14:01:22 2019 +0100 + + [ML] Tolerate total_search_time_ms not mapped in get datafeed stats (#46432) + + ML users who upgrade from versions prior to 7.4 to 7.4 or later + will have ML results indices that do not have mappings for the + total_search_time_ms field. Therefore, when searching these + indices we must tolerate this field not having a mapping. + + Fixes #46437 + +commit c2194aa7e10c97d3ebc3c6a72b8fb10c06d19679 +Author: Hendrik Muhs +Date: Fri Sep 6 14:29:46 2019 +0200 + + [Transform] simplify class structure of indexer (#46306) + + simplify transform task and indexer + + - remove redundant transform id + - moving client data frame indexer (and builder) into a separate file + +commit bb7bff5e30763aaa50d610071e7b67d9a4464675 +Author: James Rodewig +Date: Fri Sep 6 09:22:08 2019 -0400 + + [DOCS] Replace "// TESTRESPONSE" magic comments with "[source,console-result] (#46295) (#46418) + +commit 8208ffa6665461831b569c25e27ffde00a8572aa +Author: István Zoltán Szabó +Date: Fri Sep 6 15:17:18 2019 +0200 + + [DOCS] Adds progress parameter description to the GET stats data frame analytics API doc. (#46434) + +commit 323ec022bea92ea9b00377cf1431dff6699dd000 +Author: markharwood +Date: Fri Sep 6 13:59:47 2019 +0100 + + Deprecate the "index.max_adjacency_matrix_filters" index setting (#46394) + + Following performance optimisations to the adjacency_matrix aggregation we no longer require this setting. Marked as deprecated and due for removal in 8.0 + + Related #46324 + +commit a6834068e3b361be2c6b169ebbc80725ce684d71 +Author: Dimitris Athanasiou +Date: Fri Sep 6 14:13:46 2019 +0300 + + [7.x][ML] Extract DataFrameAnalyticsTask into its own class (#46402) (#46426) + + This refactors `DataFrameAnalyticsTask` into its own class. + The task has quite a lot of functionality now and I believe it would + make code more readable to have it live as its own class rather than + an inner class of the start action class. + + Backport of #46402 + +commit 27889b3d98c7aa0666ab36b51f99c42fa83f3297 +Author: lois.左 <5608413+xunux@users.noreply.github.com> +Date: Fri Sep 6 16:03:59 2019 +0800 + + Fix typo in update API example (#46397) + +commit 7582af27b079d0a7df8e11a6c9c23ffdd9adb907 +Author: Yunfeng,Wu +Date: Fri Sep 6 15:44:55 2019 +0800 + + Resolve the incorrect scroll_current when delete or close index (#45226) + + Resolve the incorrect current scroll for deleted or closed index + +commit f2a6c88f831674200ee922330680868f19376e70 +Author: Jim Ferenczi +Date: Fri Sep 6 09:29:27 2019 +0200 + + Add a system property to ignore awareness attributes (#46375) + + This is a follow up of #19191 for 7.x. + This change adds a system property called "es.routing.search_ignore_awareness_attributes" that when set to true will + effectively ignore allocation awareness attributes when routing search and get requests. This is now the default in 8.x so this + commit adds a way to opt-in to this new behavior in a minor version of 7.x. + + Relates #45735 + +commit 229086555949d31dfa0d50d33d61a82ca109dc33 +Author: Tanguy Leroux +Date: Fri Sep 6 09:10:06 2019 +0200 + + Fix usage of randomIntBetween() in testWriteBlobWithRetries (#46380) + + This commit fixes the usage of randomIntBetween() in the test + testWriteBlobWithRetries, when the test generates a random array + of a single byte. + +commit 42ada88c5dfd675daee9232e7784f93869529652 +Author: Hendrik Muhs +Date: Fri Sep 6 08:55:57 2019 +0200 + + cleanup static member + +commit ab5f09d29bb29efca6da0813d2deac9bdfdeca4e +Author: Hendrik Muhs +Date: Fri Sep 6 08:35:24 2019 +0200 + + [ML-DataFrame] improve error message for timeout case in stop (#46131) + + improve error message if stopping of transform times out. + + related #45610 + +commit 78824cce81ef2a91c940b9b137bbfa45eada3fb2 +Author: Hendrik Muhs +Date: Thu Sep 5 21:59:37 2019 +0200 + + reuse mock client to avoid probles with thread context closed errors (#46398) + +commit 327da31db44c52ec1103c3849248de1242c58c90 +Author: James Rodewig +Date: Thu Sep 5 15:03:33 2019 -0400 + + [DOCS] Reformat index stats API docs (#46322) + +commit 758680c549084742898a8634a8359dd60ff97630 +Author: Paul Sanwald +Date: Thu Sep 5 15:14:36 2019 -0400 + + version bump to 6.8.4 (#46409) + +commit 92866f977aaaeee1e88a289355fa899fd9be8ce8 +Author: Jason Tedor +Date: Thu Sep 5 15:10:56 2019 -0400 + + Clarify error message on keystore write permissions (#46321) + + When the Elasticsearch process does not have write permissions to + upgrade the Elasticsearch keystore, we bail with an error message that + indicates there is a filesystem permissions problem. This commit + clarifies that error message by pointing out the directory where write + permissions are required, or that the user can also run the + elasticsearch-keystore upgrade command manually before starting the + Elasticsearch process. In this case, the upgrade would not be needed at + runtime, so the permissions would not be needed then. + +commit 9f1339d0cebc931bc7d1c87618cc62cb8a14fa83 +Author: Lisa Cawley +Date: Thu Sep 5 11:19:35 2019 -0700 + + [DOCS] Reformats Watcher APIs using template (#46152) + +commit af1516a377432f8ac5e34c3b9d2ec0c0eddc1663 +Author: Jason Tedor +Date: Thu Sep 5 14:40:38 2019 -0400 + + Add docs on upgrading the keystore (#46331) + + This commit adds a note to the docs regarding upgrading the keystore. + +commit e078d6224720adf1b39137aeed1fb94fc1a82fc1 +Author: Lisa Cawley +Date: Thu Sep 5 10:44:21 2019 -0700 + + [DOCS] Identify reloadable Azure repository plugin settings (#46358) + +commit 457ff3e2fbd9eb0e5e4eca8282cc79f74d748633 +Author: Benjamin Trent +Date: Thu Sep 5 13:23:26 2019 -0500 + + 7.x/ml fix instance serialization bwc (#46404) + + * [ML] Fixing instance serialization version for bwc + + * fixing CppLogMessage + +commit d912a49c6fc9cfd3fee87cf381626239aa6d98ff +Author: Benjamin Trent +Date: Thu Sep 5 13:22:57 2019 -0500 + + [7.x] Support geotile_grid aggregation in composite agg sources (#45810) (#46399) + + * Support geotile_grid aggregation in composite agg sources (#45810) + + Adds support for `geotile_grid` as a source in composite aggs. + + Part of this change includes adding a new docFormat of `GEOTILE` that formats a hashed `long` value into a geotile formatting string `zoom/x/y`. + +commit 828ff01515f1adb01be57eb4d6faf5e7d13590cf +Author: Lisa Cawley +Date: Thu Sep 5 11:12:39 2019 -0700 + + [DOCS] Update snippets in security APIs (#46191) (#46401) + +commit caf3e4d65432077af9a5e19b757bc5e75ee9fdf6 +Author: Benjamin Trent +Date: Thu Sep 5 10:31:11 2019 -0500 + + [7.x] [ML][Transforms] fixing rolling upgrade continuous transform test (#45823) (#46347) (#46337) + + * [ML][Transforms] fixing rolling upgrade continuous transform test (#45823) + + * [ML][Transforms] fixing rolling upgrade continuous transform test + + * adjusting wait assert logic + + * adjusting wait conditions + + * [ML][Transforms] allow executor to call start on started task (#46347) + + * making sure we only upgrade from 7.4.0 in test + +commit bac5cb1c7aeef34a433c11f88d6041cef7593488 +Author: Benjamin Trent +Date: Thu Sep 5 10:30:40 2019 -0500 + + [ML][Transforms] update supported aggs docs (#46388) (#46392) + +commit 96a4dc21e09cdcfe644c21d53fb3e7c35d279d21 +Author: James Rodewig +Date: Thu Sep 5 10:18:15 2019 -0400 + + [DOCS] Re-add versioning to put template docs (#46384) + + Adds documentation for index template versioning + accidentally removed with #46297. + +commit 1f36c4e50c9bdd5f6b2c68ae2ab533eb4e91786b +Author: James Rodewig +Date: Thu Sep 5 10:11:25 2019 -0400 + + [DOCS] Replace "// CONSOLE" comments with [source,console] (#46159) (#46332) + +commit d5597c9ba6e5e90e9b1c850a7398eeeb1bef71f6 +Author: James Rodewig +Date: Thu Sep 5 08:33:17 2019 -0400 + + [DOCS] Reformat index segments API docs (#46345) + +commit 3f5f028e2cf1f902dece4a8a79b5f4c8b040d49e +Author: Adrien Grand +Date: Wed Sep 4 17:07:51 2019 +0200 + + Fix release version of #38095. + +commit 7a9af874ad8c2c545791b5cde99c693b0ae53aa4 +Author: Armin Braun +Date: Thu Sep 5 14:03:38 2019 +0200 + + Enable Debug Logging for Master and Coordination Packages (#46363) (#46374) + + In order to track down #46091: + * Enables debug logging in REST tests for `master` and `coordination` packages + since we suspect that issues are caused by failed and then retried publications + +commit 5201386232f857f55b46ea4b09b4bb71e75b8181 +Author: Benjamin Trent +Date: Thu Sep 5 06:57:33 2019 -0500 + + [ML] testFullClusterRestart waiting for stable cluster (#46280) (#46335) + + * [ML] waiting for ml indices before waiting task assignment testFullClusterRestart + + * waiting for a stable cluster after fullrestart + + * removing unused imports + +commit 7e4c633ce38807847fe1c8006e7d295b9a310ff1 +Author: Yannick Welsch +Date: Thu Sep 5 13:03:00 2019 +0200 + + Quiet down shard lock failures (#46368) + + These were actually never intended to be logged at the warning level but made visible by a refactoring in #19991, which introduced a new exception type but forgot to adapt some of the consumers of the exception. + +commit 999658826f4ee3be74cb6361268e3fff0c255e38 +Author: Ioannis Kakavas +Date: Thu Sep 5 12:24:19 2019 +0300 + + Mute failing SamlAuthenticationIT tests (#46369) + + see #44410 + +commit 28974b5723a2caa991dafb486715f5016402a6b7 +Author: Tanguy Leroux +Date: Thu Sep 5 09:25:23 2019 +0200 + + Replace mocked client in GCSBlobStoreRepositoryTests by HTTP server (#46255) + + This commit removes the usage of MockGoogleCloudStoragePlugin in + GoogleCloudStorageBlobStoreRepositoryTests and replaces it by a + HttpServer that emulates the Storage service. This allows the repository + tests to use the real Google's client under the hood in tests and will allow + us to test the behavior of the snapshot/restore feature for GCS repositories + by simulating random server-side internal errors. + + The HTTP server used to emulate the Storage service is intentionally simple + and minimal to keep things understandable and maintainable. Testing full + client options on the server side (like authentication, chunked encoding + etc) remains the responsibility of the GoogleCloudStorageFixture. + +commit 6d1a82134cff771eaa1bf9f0b8ed9bc7308f91de +Author: Tanguy Leroux +Date: Thu Sep 5 09:22:42 2019 +0200 + + Add repository integration tests for Azure (#46263) + + Similarly to what had been done for S3 (#46081) and GCS (#46255) + this commit adds repository integration tests for Azure, based on an + internal HTTP server instead of mocks. + +commit 9563d65d1ed5dff8119efcf66e61eb89da6dd0d1 +Author: Przemyslaw Gomulka +Date: Thu Sep 5 08:50:35 2019 +0200 + + Improve documentation for X-Opaque-ID (#46167) (#46333) + + this field can be present in search slow logs and deprecation logs. The + docs describes how to enable this functionality and what expect in logs. + closes #44851 + +commit d5acb15a71a55d907014a397de4db5021d09ac95 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Sep 5 14:34:19 2019 +1000 + + [Backport] Initialize document subset bit set cache used for DLS (#46211) (#46359) + + This commit initializes DocumentSubsetBitsetCache even if DLS + is disabled. Previously it would throw null pointer when querying + usage stats if we explicitly disabled DLS as there would be no instance of DocumentSubsetBitsetCache to query. It is okay to initialize + DocumentSubsetBitsetCache which will be empty as the license enforcement + would prevent usage of DLS feature and it will not fail when accessing usage stats. + + Closes #45147 + +commit 03ed18a0102fb87bf4dc22c580831c8b7a924815 +Author: Nhat Nguyen +Date: Wed Sep 4 17:23:41 2019 -0400 + + Unmute testRecoveryFromFailureOnTrimming + + Tracked at #46267 + +commit b11968cf4d9a447c71c14c2af377bad4731a438c +Author: Lisa Cawley +Date: Wed Sep 4 17:03:05 2019 -0700 + + [DOCS] Synchs Watcher API titles with better HLRC titles (#46328) + +commit da5618cbf4c48a380ef20ad90fc5f9b569454bcc +Author: Lisa Cawley +Date: Wed Sep 4 16:24:55 2019 -0700 + + [DOCS] Identify reloadable GCS repository plugin settings (#46352) + +commit 9faf1d956f4b73326fe9ea13dc344f429c687c7e +Author: Lisa Cawley +Date: Wed Sep 4 14:43:58 2019 -0700 + + [DOCS] Identify reloadable S3 repository plugin settings (#46349) + +commit 40c3225d266df98483dc4f01dec467b80e173440 +Author: Julie Tibshirani +Date: Wed Sep 4 14:45:57 2019 -0700 + + First round of optimizations for vector functions. (#46294) + + This PR merges the `vectors-optimize-brute-force` feature branch, which makes + the following changes to how vector functions are computed: + * Precompute the L2 norm of each vector at indexing time. (#45390) + * Switch to ByteBuffer for vector encoding. (#45936) + * Decode vectors and while computing the vector function. (#46103) + * Use an array instead of a List for the query vector. (#46155) + * Precompute the normalized query vector when using cosine similarity. (#46190) + + Co-authored-by: Mayya Sharipova + +commit a16cb89956fffbcc00d50936dd27c4b34c34020b +Author: Nhat Nguyen +Date: Wed Sep 4 16:50:34 2019 -0400 + + Revert "Sync translog without lock when trim unreferenced readers (#46203)" + + Unfortunately, with this change, we won't clean up all unreferenced + generations when reopening. We assume that there's at most one + unreferenced generation when reopening translog. The previous + implementation guarantees this assumption by syncing translog every time + after we remove a translog reader. This change, however, only syncs + translog once after we have removed all unreferenced readers (can be + more than one) and breaks the assumption. + + Closes #46267 + + This reverts commit fd8183ee51d7cf08d9def58a2ae027714beb60de. + +commit d5ad86dad923110534f539993afb645b0506a6b0 +Author: Nik Everett +Date: Wed Sep 4 15:19:20 2019 -0400 + + Build: Enable testing without magic comments (backports #46180) (#46325) + + Previously we only turned on tests if we saw either `// CONSOLE` or + `// TEST`. These magic comments are difficult for the docs build to deal + with so it has moved away from using them where possible. We should + catch up. This adds another trigger to enable testing: marking a snippet + with the `console` language. It looks like this: + + ``` + [source,console] + ---- + GET / + ---- + ``` + + This saves a line which is nice, I guess. But it is more important to me + that this is consistent with the way the docs build works now. + + Similarly this enables response testing when you mark a snippet with the + language `console-result`. That looks like: + ``` + [source,console-result] + ---- + { + "result": "0.1" + } + ---- + ``` + + `// TESTRESPONSE` is still available for situations like `// TEST`: when + the response isn't *in* the console-result language (like `_cat`) or + when you want to perform substitutions on the generated test. + + Should unblock #46159. + +commit 5ee336ff78aef933f3a1499be21aa3e1478d8f71 +Author: Aleh Zasypkin +Date: Wed Sep 4 20:50:15 2019 +0200 + + [7.x] Document support of OIDC Implicit flow in Kibana. (#46329) + +commit bd7f9d5bcdd1ff98322bfd33af5f79f2ddcae014 +Author: James Rodewig +Date: Wed Sep 4 13:42:24 2019 -0400 + + [DOCS] Fix upgrade paths for 7.5 (#46150) + +commit 9f9dbcf4b7220212bc6fe298dbfc2639c4416855 +Author: Lisa Cawley +Date: Wed Sep 4 10:12:03 2019 -0700 + + [DOCS] Identify reloadable EC2 Discovery Plugin settings (#46102) + +commit f4dd3ccf32a60f2055e002817b289be883e5652c +Author: James Rodewig +Date: Wed Sep 4 11:47:32 2019 -0400 + + [DOCS] Reformat "put index template" API docs (#46297) + +commit ded98e50b79f4b6d8881506f95fb740ccbf842d0 +Author: Martijn van Groningen +Date: Wed Sep 4 15:05:27 2019 +0200 + + Change exact match processor to match processor. (#46041) + + Besides a rename, this changes allows to processor to attach multiple + enrich docs to the document being ingested. + + Also in order to control the maximum number of enrich docs to be + included in the document being ingested, the `max_matches` setting + is added to the enrich processor. + + Relates #32789 + +commit 3cbdd84b8916ee28f0ed613a1897be2fb5c3163b +Author: Jason Tedor +Date: Wed Sep 4 11:52:42 2019 -0400 + + Add test that get triggers shard search active (#46317) + + This commit is a follow-up to a change that fixed that multi-get was not + triggering a shard to become search active. In that change, we added a + test that multi-get properly triggers a shard to become search + active. This commit is a follow-up to that change which adds a test for + the get case. While get is already handled correctly in production code, + there was not a test for it. This commit adds one. Additionally, we + factor all the search idle tests from IndexShardIT into a separate test + class, as an effort to keep related tests together instead of a single + large test class containing a jumble of tests, and also to keep test + classes smaller for better parallelization. + +commit 408b58dd9d08df454599daed5aac61c535a44fe0 +Author: markharwood +Date: Wed Sep 4 16:45:32 2019 +0100 + + Adjacency_matrix aggregation optimisation. (#46257) (#46315) + + Avoid pre-allocating ((N * N) - N) / 2 “BitsIntersector” objects given N filters. + Most adjacency matrices will be sparse and we typically don’t need to allocate all of these objects - can save a lot of allocations when the number of filters is high. + + Closes #46212 + +commit 39e81c3ca61f510129f06b42a918ca475a9ac377 +Author: David Turner +Date: Wed Sep 4 16:37:00 2019 +0100 + + Docs for translog, history retention and flushing (#46245) + + This commit updates the docs about translog retention and flushing to reflect + recent changes in how peer recoveries work. It also adds some docs to describe + how history is retained for replay using soft deletes and shard history + retention leases. + + Relates #45473 + +commit eb56d23421a54c8e52f45cc442e8409edccdfd07 +Author: Nhat Nguyen +Date: Wed Sep 4 11:01:01 2019 -0400 + + Do not send recovery requests with CancellableThreads (#46287) + + Previously, we send recovery requests using CancellableThreads because + we send requests and wait for responses in a blocking manner. With async + recovery, we no longer need to do so. Moreover, if we fail to submit a + request, then we can release the Store using an interruptible thread + which can risk invalidating the node lock. + + This PR is the first step to avoid forking when releasing the Store. + + Relates #45409 + Relates #46178 + +commit 1a29711b06a17ed134e61a54a1b5deabb0204e66 +Author: Albert Zaharovits +Date: Wed Sep 4 12:10:23 2019 +0300 + + DOCS Link to kib reference from es reference on PKI authn (#46260) + +commit 034a37aa01b21a1a34d13183f04bdcad5c1855cd +Author: James Rodewig +Date: Wed Sep 4 10:58:24 2019 -0400 + + [DOCS] Add "get index template" API docs (#46296) + +commit 0f0b77b2630426dc00b1af08dd65975a80a64063 +Author: István Zoltán Szabó +Date: Wed Sep 4 15:12:49 2019 +0200 + + [DOCS] Reformats search template and multi search template APIs (#46236) + + * [DOCS] Reformats search template and multi search template APIs. + Co-Authored-By: James Rodewig + +commit 5066835569b81157d3e54ecb1ffdc49abbeebd10 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Sep 4 14:07:26 2019 +0200 + + Fix SearchService.createContext exception handling (#46258) + + An exception from the DefaultSearchContext constructor could leak a + searcher, causing future issues like shard lock obtained exceptions. The + underlying cause of the exception in the constructor has been fixed, but + as a safety precaution we also fix the exception handling in + createContext. + + Closes #45378 + +commit c71d959d6132a5b0da6472374008660e6dff2cf2 +Author: István Zoltán Szabó +Date: Wed Sep 4 11:34:30 2019 +0200 + + [DOCS] Reformats search shards API (#46240) + + * [DOCS] Reformats search shards API + Co-Authored-By: James Rodewig + +commit d709a5c1934dadcf9329d1a06f4fcdcb5dc87110 +Author: Alpar Torok +Date: Wed Sep 4 12:18:23 2019 +0300 + + Quote the task name in reproduction line printer (#46266) + + Some tasks have `#` for instance that doesn't play well with some shells + ( e.x. zsh ) + +commit 6bec63fdfa066520c76e7b9afd1e6fb7f15127b7 +Author: Martijn van Groningen +Date: Wed Sep 4 11:17:10 2019 +0200 + + removed redundant cast + +commit 5c5af7756594842dd0f2adf37fdd94e434ce3ae3 +Author: István Zoltán Szabó +Date: Wed Sep 4 10:52:17 2019 +0200 + + [DOCS] Reformats request body search API (#46254) + + * [DOCS] Reformats request body search API. + Co-Authored-By: James Rodewig + +commit ece9eb4acdc560d2e9827203386d5a9315daf23b +Author: Andrey Ershov +Date: Wed Sep 4 10:48:32 2019 +0200 + + Remove stack trace logging in Security(Transport|Http)ExceptionHandler (#45966) + + As per #45852 comment we no longer need to log stack-traces in + SecurityTransportExceptionHandler and SecurityHttpExceptionHandler even + if trace logging is enabled. + + (cherry picked from commit c99224a32d26db985053b7b36e2049036e438f97) + +commit f2bdd392e76382bd0a8ba393e7b170fc91628d57 +Author: István Zoltán Szabó +Date: Wed Sep 4 10:14:30 2019 +0200 + + [DOCS] Reformats multi search API (#46256) + + * [DOCS] Reformats multi search API. + + Co-Authored-By: James Rodewig + +commit 8fca5b52049b977d197b1ee0bbf6109f8d1b0639 +Author: Dimitris Athanasiou +Date: Wed Sep 4 10:57:01 2019 +0300 + + [7.x][ML] Unmute testStopOutlierDetectionWithEnoughDocumentsToScroll (#46271) (#46282) + + The test seems to have been failing due to a race condition between + stopping the task and refreshing the destination index. In particular, + we were going forward with refreshing the destination index even + though the task stopped in the meantime. This was fixed in + request. + + Closes #43960 + + Backport of #46271 + +commit fd0affb503cda6f30265aec90d11cfdc0f5a9e5e +Author: Marios Trivyzas +Date: Wed Sep 4 10:32:01 2019 +0300 + + SQL: Fix issue with IIF function when condition folds (#46290) + + Previously, when the condition (1st argument) of the IIF function could + be evaluated (folded) to false, the `IfConditional` was eliminated which + caused `IndexOutOfBoundsException` to be thrown when `info()` and + `resolveType()` methods where called. + + Fixes: #46268 + + (cherry picked from commit 9a885a3ac47bc8f52c07770d1d8d670ce0af1e59) + +commit 3f67cbe974b8e9ce64d3c2ce7dd1c991360befa7 +Author: Nhat Nguyen +Date: Tue Sep 3 12:09:08 2019 -0400 + + Suppress warning from background sync on relocated primary (#46247) + + If a primary as being relocated, then the global checkpoint and + retention lease background sync can emit unnecessary warning logs. + This side effect was introduced in #42241. + + Relates #40800 + Relates #42241 + +commit 5924df17645590f8d80bb52413361304d4cee0d2 +Author: Nhat Nguyen +Date: Tue Sep 3 14:51:16 2019 -0400 + + Mute testRecoveryFromFailureOnTrimming + + Tracked at #46267 + +commit f91dd903cb1c01a5a06d74a30d15853e0a39b0b4 +Author: Nhat Nguyen +Date: Tue Sep 3 12:31:24 2019 -0400 + + Strengthen testUpdate in rolling upgrade + + We hit a bug where we can't partially update documents created in a + mixed cluster between 5.x and 6.x. Although this bug does not affect + 7.0 or later, we should have a good test that catches this issue. + + Relates #46198 + +commit 57f322f85eba7385af5686ab83e5739323ba9c2b +Author: Lee Hinman +Date: Tue Sep 3 16:19:25 2019 -0600 + + Move MockRespository into test framework (#46298) + + This moves the `MockRespository` class into `test/framework/src/main` so + it can be used across all modules and plugins in tests. + +commit 8a4ff7d57d318e2e7f3eba1ca4db632d37cb3647 +Author: Benjamin Trent +Date: Tue Sep 3 15:27:24 2019 -0500 + + [ML][Transforms] protecting doSaveState with optimistic concurrency (#46156) (#46281) + + * [ML][Transforms] protecting doSaveState with optimistic concurrency + + * task code cleanup + +commit c3c1dcd2ac62c7a2bef38604cb1a64464f9afb0f +Author: Benjamin Trent +Date: Tue Sep 3 15:26:56 2019 -0500 + + [ML][Transforms] fixing listener being called twice (#46284) (#46292) + +commit b8c51ff8942742de29d4cd24c61db9d892a73068 +Author: Jason Tedor +Date: Tue Sep 3 14:28:21 2019 -0400 + + Multi-get requests should wait for search active (#46283) + + When a shard has fallen search idle, and a non-realtime multi-get + request is executed, today such requests do not wait for the shard to + become search active and therefore such requests do not wait for a + refresh to see the latest changes to the index. This also prevents such + requests from triggering the shard as non-search idle, influencing the + behavior of scheduled refreshes. This commit addresses this by attaching + a listener to the shard search active state for multi-get requests. In + this way, when the next scheduled refresh is executed, the multi-get + request will then proceed. + +commit 85e2e41de77bfa3a6ce33259b41255d63c3926c4 +Author: Zachary Tong +Date: Tue Sep 3 12:10:34 2019 -0400 + + Add CumulativeCard pipeline agg to pipeline index (#46279) + + The Cumulative Cardinality docs weren't linked + from the pipeline index page + +commit 53df54c7036a34b54b6cdacce0e5dc473cd8a1db +Author: Benjamin Trent +Date: Tue Sep 3 11:04:18 2019 -0500 + + [ML][Transforms] fixing stop on changes check bug (#46162) (#46273) + + * [ML][Transforms] fixing stop on changes check bug + + * Adding new method finishAndCheckState to cover race conditions in early terminations + + * changing stopping conditions in `onStart` + + * allow indexer to finish when exiting early + +commit 2383acaa89f82a5a0a7272d7f0e48f6721f75e9a +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Sep 3 17:29:28 2019 +0200 + + Fix testSyncFailsIfOperationIsInFlight (#46269) + + testSyncFailsIfOperationIsInFlight could fail due to the index request + spawing a GCP sync (new since 7.4). Test now waits for it to finish + before testing that flushed sync fails. + +commit 3d4b8e01c7f6906ffd0c65e0788c7ef7a54f13c9 +Author: Lee Hinman +Date: Tue Sep 3 09:20:02 2019 -0600 + + Validate SLM policy ids strictly (#45998) (#46145) + + This uses strict validation for SLM policy ids, similar to what we use + for index names. + + Resolves #45997 + +commit f77aed8ea926d3d57dac873453d2553653b5cd76 +Author: James Rodewig +Date: Tue Sep 3 09:10:46 2019 -0400 + + [DOCS] Add delete index alias API docs (#46080) + +commit ab045744ac456c204d158d77c45f671216c3ae18 +Author: David Roberts +Date: Tue Sep 3 12:41:44 2019 +0100 + + [ML-DataFrame] Fix off-by-one error in checkpoint operations_behind (#46235) + + Fixes a problem where operations_behind would be one less than + expected per shard in a new index matched by the data frame + transform source pattern. + + For example, if a data frame transform had a source of foo* + and a new index foo-new was created with 2 shards and 7 documents + indexed in it then operations_behind would be 5 prior to this + change. + + The problem was that an empty index has a global checkpoint + number of -1 and the sequence number of the first document that + is indexed into an index is 0, not 1. This doesn't matter for + indices included in both the last and next checkpoints, as the + off-by-one errors cancelled, but for a new index it affected + the observed result. + +commit 59c474e67523ea3f538a217bb1300b3af5e45600 +Author: Ignacio Vera +Date: Tue Sep 3 10:24:46 2019 +0200 + + reset queryGeometry in ShapeQueryTests (#45974) (#46251) + +commit 416419e4c9e453dbe03071b6f1f5d1c756341a17 +Author: dengweisysu +Date: Tue Sep 3 02:56:22 2019 +0800 + + Sync translog without lock when trim unreferenced readers (#46203) + + With this change, we can avoid blocking writing threads when trimming + unreferenced readers; hence improving the translog writing performance + in async durability mode. + + Close #46201 + +commit e01ec802e70a919c3d80b42939491eb6d78f0a2e +Author: Anup +Date: Tue Sep 3 04:59:21 2019 +0530 + + Remove duplicate line in SearchAfterBuilder (#45994) + +commit 2662c1b41743dbf03087a05c29eab199e16546b3 +Author: Armin Braun +Date: Mon Sep 2 18:04:37 2019 +0200 + + Wait for all Rec. to Stop on Node Close (#46178) (#46237) + + * Wait for all Rec. to Stop on Node Close + + * This issue is in the `RecoverySourceHandler#acquireStore`. If we submit the store release to the generic threadpool while it is getting shut down we never complete the futue we wait on (in the generic pool as well) and fail to ever release the store potentially. + * Fixed by waiting for all recoveries to end on node close so that we aways have a healthy thread pool here + * Closes #45956 + +commit bd7a04cd553c131932405bb2e927b75a0f4691c2 +Author: Tanguy Leroux +Date: Mon Sep 2 16:41:18 2019 +0200 + + Disable request throttling in S3BlobStoreRepositoryTests (#46226) + + When some high values are randomly picked up - for example the number + of indices to snapshot or the number of snapshots to create - the tests + in S3BlobStoreRepositoryTests can generate a high number of requests to + the internal S3 server. + + In order to test the retry logic of the S3 client, the internal server is + designed to randomly generate random server errors. When many + requests are made, it is possible that the S3 client reaches its maximum + number of successive retries capacity. Then the S3 client will stop + retrying requests until enough retry attempts succeed, but it means + that any request could fail before reaching the max retries count and + make the test fail too. + + Closes #46217 + Closes #46218 + Closes #46219 + +commit 3513eaeed8f90cb76725d279bc57190bbf6f4376 +Author: Ioannis Kakavas +Date: Mon Sep 2 16:53:46 2019 +0300 + + Re-enable bwc testing + +commit a9e3871fc7e0a7c1de94deed34e92e1fb9a64a8b +Author: markharwood +Date: Mon Sep 2 13:31:22 2019 +0100 + + Test fix for PinnedQueryBuilderIT (#46187) (#46227) + + Fix test issue to stabilise scoring through use of DFS search mode. + Randomised index-then-delete docs introduced by the test framework likely caused an imbalance in IDF scores across shards. Also made number of shards used in test a random number for added test coverage. + + Closes #46174 + +commit d68e05aade2d88332f625c61088628462eac57e2 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Sep 2 10:37:32 2019 +0200 + + Mute 2 tests in S3BlobStoreRepositoryTests (#46221) + + Muted testSnapshotAndRestore and testMultipleSnapshotAndRollback + + Relates #46218 and #46219 + +commit 3bee647e5b10579c6d7dd6729f06ca64be4b6240 +Author: Marios Trivyzas +Date: Mon Sep 2 11:16:04 2019 +0300 + + SQL: Fix issue with DataType for CASE with NULL (#46173) + + Previously, if the DataType of all the WHEN conditions of a CASE + statement is NULL, then it was set to NULL even if the ELSE clause + has a non-NULL data type, e.g.: + ``` + CASE WHEN a = 1 THEN NULL + WHEN a = 5 THEN NULL + ELSE 'foo' + ``` + + Fixes: #46032 + (cherry picked from commit 8c1012efbbd3a300afd0dfb9b18250f15ea753f9) + +commit 555b6301606c7a2b14da8389570972af37a7190c +Merge: 51a703da29d 5747badaa8b +Author: Martijn van Groningen +Date: Mon Sep 2 09:16:55 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 5747badaa8b85f6316ce6e40a23713faedea0348 +Author: Martijn van Groningen +Date: Mon Sep 2 08:17:03 2019 +0200 + + Allow ingest processors access to node client. (#46077) + + This is the first PR that merges changes made to server module from + the enrich branch (see #32789) into the master branch. + + The plan is to merge changes made to the server module separately from + the pr that will merge enrich into master, so that these changes can + be reviewed in isolation. + +commit 6fed08214831ef389e21e80603bff699ce1f5eb6 +Author: Jay Greenberg +Date: Sat Aug 31 12:14:34 2019 -0400 + + Clarify default behavior of auto_create_index (#46134) + + Be specific about the default behaviour of `action.auto_create_index` when a list is given. + +commit a078bb4b92cc13f96a329b7cf094237596445ed1 +Author: Ryan Ernst +Date: Fri Aug 30 11:37:18 2019 -0700 + + Add test tasks for unpooled and direct buffer pooling to netty (#46049) + + Some netty behavior is controlled by system properties. While we want to + test with the defaults for Elasticsearch for most tests, within netty we + want to ensure these netty settings exhibit correct behavior. This + commit adds variants of test and integTest tasks for netty which set the + unpooled and direct buffer pooled allocators. + + relates #45881 + +commit b89b0d5f8421cf2cc3e21e1a483841d1c6c97782 +Author: Jason Tedor +Date: Fri Aug 30 11:00:42 2019 -0400 + + Move plugin.mandatory to its own page + + This commit takes the reworking of plugin.mandatory docs even farther by + taking this setting to its own page. + +commit 196e0c78c0baea7de1e6d79fb7773f996d937547 +Author: Jason Tedor +Date: Fri Aug 30 10:29:14 2019 -0400 + + Move plugin.mandatory to installing plugins docs + + This commit moves the plugin.mandatory settings from the plugin + directory page in the docs to the installing plugins page in the docs. + +commit d0c5573a51ce2bd9fd27e3fa398dbab3e7f36f58 +Author: Benjamin Trent +Date: Fri Aug 30 09:27:07 2019 -0500 + + [ML] Throw an error when a datafeed needs CCS but it is not enabled for the node (#46044) (#46096) + + Though we allow CCS within datafeeds, users could prevent nodes from accessing remote clusters. This can cause mysterious errors and difficult to troubleshoot. + + This commit adds a check to verify that `cluster.remote.connect` is enabled on the current node when a datafeed is configured with a remote index pattern. + +commit db949847e5902e93a8c78beaa928fb607ed4bc63 +Author: Nhat Nguyen +Date: Fri Aug 30 08:42:00 2019 -0400 + + Fix translog stats in testPrepareIndexForPeerRecovery (#46137) + + When recovering a shard locally, we use a translog snapshot from + newSnapshotFromGen which consists of all readers from a certain + generation. In the test, we use newSnapshotFromMinSeqNo for the + expectation. The snapshot of this method includes only readers + containing operations in the requesting range. + + Closes #46022 + +commit 98c32c7846a6f96268b6569aec580b40cfc5db8e +Author: Alexander Reelsen +Date: Fri Aug 30 14:02:24 2019 +0200 + + Fix wrong URL encoding in watcher HTTP client (#45894) + + The test assumption was calling the wrong method resulting in a URL + encoding before returning the data. + + Closes #44970 + +commit 53f70ee99647e5fecc141a365276c4e2dda5d356 +Author: István Zoltán Szabó +Date: Thu Aug 29 10:04:49 2019 +0200 + + [DOCS] Reformats URI search request (#45844) + + * [DOCS] Reformats URI search request. + + Co-Authored-By: James Rodewig + + Co-Authored-By: debadair + +commit 152ce62c5880843df50071d57ab6fcd5bcad29ab +Author: Andrey Ershov +Date: Fri Aug 30 12:59:16 2019 +0200 + + Enhanced logging when transport is misconfigured to talk to HTTP port (#45964) + + If a node is misconfigured to talk to remote node HTTP port (instead of + transport port) eventually it will receive an HTTP response from the + remote node on transport port (this happens when a node sends + accidentally line terminating byte in a transport request). + If this happens today it results in a non-friendly log message and a + long stack trace. + This commit adds a check if a malformed response is HTTP response. In + this case, a concise log message would appear. + + (cherry picked from commit 911d02b7a9c3ce7fe316360c127a935ca4b11f37) + +commit 0c1b263e8d08c2b83bfffddfd0c592d4cf6caf62 +Author: Tanguy Leroux +Date: Fri Aug 30 10:00:32 2019 +0200 + + Inject random errors in S3BlobStoreRepositoryTests (#46125) + + This commit modifies the HTTP server used in S3BlobStoreRepositoryTests + so that it randomly returns server errors for any type of request executed by + the SDK client. It is now possible to verify that the repository tests are s + uccessfully completed even if one or more errors were returned by the S3 + service in response of a blob upload, a blob deletion or a object listing request + etc. + + Because injecting errors forces the SDK client to retry requests, the test limits + the maximum errors to send in response for each request at 3 retries. + +commit 57ac4f391b64a9e72544c0504d6862b4151be83a +Author: Colin Goodheart-Smithe +Date: Fri Aug 30 09:08:42 2019 +0100 + + Adds release notes for 7.4.0 + +commit 5921ae53d807c387ebb1ff60dd40c589c50f2bfe +Author: Dimitris Athanasiou +Date: Fri Aug 30 09:57:43 2019 +0300 + + [7.x][ML] Regression dependent variable must be numeric (#46072) (#46136) + + * [ML] Regression dependent variable must be numeric + + This adds a validation that the dependent variable of a regression + analysis must be numeric. + + * Address review comments and fix some problems + + In addition to addressing the review comments, this + commit fixes a few issues I found during testing. + + In particular: + + - if there were mappings for required fields but they were + not included we were not reporting the error + - if explicitly included fields had unsupported types we were + not reporting the error + + Unfortunately, I couldn't get those fixed without refactoring + the code in `ExtractedFieldsDetector`. + +commit e25f1ac8a229e9e0993b2bf78b1ab69bf3a55660 +Author: Ryan Ernst +Date: Thu Aug 29 14:13:35 2019 -0700 + + Add package docs for bundled jdk location (#46153) + + This commit expands the documented directory layout of the rpm and deb + packages to include the bundled jdk. + + closes #45150 + +commit 2aafe8aecba5ec2ad0bef32f18f4f00590c42d6a +Author: James Rodewig +Date: Thu Aug 29 15:10:10 2019 -0400 + + [DOCS] Bump docs version to 7.5.0 (#46143) + +commit 8bdbc7d9bf03254a15b26b83c23e0ae7b7cde93a +Author: Paul Sanwald +Date: Thu Aug 29 15:03:26 2019 -0400 + + Bump version from 7.4 to 7.5 (#46142) + +commit e7d33cc1c6f6991215b690493525c49d66cb5515 +Author: James Rodewig +Date: Thu Aug 29 14:37:47 2019 -0400 + + [DOCS] Add "delete index template" API docs (#46101) + +commit b5d8b364bbcac4f9e29e33440540a6f8462fb39f +Author: Julie Tibshirani +Date: Thu Aug 29 10:56:53 2019 -0700 + + Ensure top docs optimization is fully disabled for queries with unbounded max scores. (#46105) (#46139) + + When a query contains a mandatory clause that doesn't track the max score per + block, we disable the max score optimization. Previously, we were doing this by + wrapping the collector with a FilterCollector that always returned + ScoreMode.COMPLETE. + + However we weren't adjusting totalHitsThreshold, so the collector could still + call Scorer#setMinCompetitiveScore. It is against the method contract to call + setMinCompetitiveScore when the score mode is COMPLETE, and some scorers like + ReqOptSumScorer throw an error in this case. + + This commit tries to disable the optimization by always setting + totalHitsThreshold to max int, as opposed to wrapping the collector. + +commit c4aa8e0d906a03879e6e1883c7137fb6c19d160c +Author: James Rodewig +Date: Thu Aug 29 13:18:47 2019 -0400 + + [DOCS] Add "index template exists" API docs (#46095) + +commit bd5084585531f46ceb9bd415f527ef38a61b5c86 +Author: debadair +Date: Thu Aug 29 09:56:10 2019 -0700 + + [DOCS] Backporting Reformats delete by query API (#46051) (#46138) + + * [DOCS] Reformats delete by query API (#46051) + + * Reformats delete by query API + + * Update docs/reference/docs/delete-by-query.asciidoc + + Co-Authored-By: James Rodewig + + * Updated common parms includes. + + * [DOCS] Fixed issue in Common Parms. + +commit cf8a4171e12a7a12e94e7e8638fdf9ed49dff96a +Author: Zachary Tong +Date: Thu Aug 29 12:45:39 2019 -0400 + + Rename `data-science` plugin to `analytics` (#46133) + + Rename `data-science` plugin to `analytics`. + Also removes enabled flag. Backport of #46092 + +commit 9b2ea07b175126fd55cc1d8025d985bea74fde6b +Author: Simon Willnauer +Date: Thu Aug 29 17:54:15 2019 +0200 + + Flush engine after big merge (#46066) (#46111) + + Today we might carry on a big merge uncommitted and therefore + occupy a significant amount of diskspace for quite a long time + if for instance indexing load goes down and we are not quickly + reaching the translog size threshold. This change will cause a + flush if we hit a significant merge (512MB by default) which + frees diskspace sooner. + +commit f7a91239e78902554afea00af60b0b3baf39d873 +Author: James Rodewig +Date: Thu Aug 29 11:18:18 2019 -0400 + + [DOCS] Reformat update index aliases API docs (#46093) + +commit 79f26f83082c551e8a557398f0c7790e5972be2d +Author: James Rodewig +Date: Thu Aug 29 10:38:48 2019 -0400 + + [DOCS] Separate add index alias API docs (#46086) + +commit 51a703da29d265d0a4edb5a9e03b5908e7e587e4 +Author: Michael Basnight +Date: Thu Aug 29 09:10:07 2019 -0500 + + Add enrich transport client support (#46002) + + This commit adds an enrich client, as well as a smoke test to validate + the client works. + +commit 3e62cf9d74a4d80948d73a32cd937d80cca58834 +Author: James Rodewig +Date: Thu Aug 29 10:07:52 2019 -0400 + + [DOCS] Correct custom analyzer callouts (#46030) + +commit 322d95f2f6cf0f0eef76e951da4c733f69281bf1 +Author: James Rodewig +Date: Thu Aug 29 09:44:58 2019 -0400 + + [DOCS] Add get index alias API docs (#46046) + +commit bb49124690ebd84361c10ac68a27e5636eb8aff1 +Author: Nhat Nguyen +Date: Thu Aug 29 08:43:24 2019 -0400 + + Only verify global checkpoint if translog sync occurred (#45980) + + We only sync translog if the given offset hasn't synced yet. We can't + verify the global checkpoint from the latest translog checkpoint unless + a sync has occurred. + + Closes #46065 + Relates #45634 + +commit 028e792e1d7ef7ba19cc35fd9fa3ce1ef98bda05 +Author: Nhat Nguyen +Date: Tue Aug 27 12:36:25 2019 -0400 + + Remove already exist assertion while renew ccr lease (#46009) + + If a CCR lease is disappeared while we are renewing it, then we will + issue asyncAddRetentionLease to add that lease. And if + asyncAddRetentionLease takes longer than retentionLeaseRenewInterval, + then we can issue another asyncAddRetentionLease request. One of + asyncAddRetentionLease requests will fail with + RetentionLeaseAlreadyExistsException, hence trip the assertion. + + Closes #45192 + +commit 8145845fca76079f07227ad5419d629c84e1decc +Author: James Rodewig +Date: Thu Aug 29 09:12:16 2019 -0400 + + [DOCS] Reformats analyze API (#45986) + +commit a75348d1fb1d5104b87f6a37559706dc94bf75bc +Author: István Zoltán Szabó +Date: Thu Aug 29 14:38:14 2019 +0200 + + [DOCS] [PUT DFA] Documents inline the child params of source and dest (#45649) + + * [DOCS] [PUT DFA] Documents inline the child params of source and dest. + + * [DOCS] Fixes indentation issues and amends dfa definitions. + +commit d9be906afb15fe3575e9739e0a2c1569eb40e425 +Author: Jason Tedor +Date: Thu Aug 29 08:53:17 2019 -0400 + + Start testing against AdoptOpenJDK (#45666) + + This commit adds AdoptOpenJDK to the testing matrix. + +commit b8a0379057362943fd40b6a6a2abecfc9f45ee7e +Author: Przemysław Witek +Date: Thu Aug 29 14:21:03 2019 +0200 + + Refactor auditor-related classes (#45893) (#46120) + +commit d340530a4753b66da75766b752dada943029ec78 +Author: David Turner +Date: Thu Aug 29 12:39:28 2019 +0100 + + Avoid overshooting watermarks during relocation (#46079) + + Today the `DiskThresholdDecider` attempts to account for already-relocating + shards when deciding how to allocate or relocate a shard. Its goal is to stop + relocating shards onto a node before that node exceeds the low watermark, and + to stop relocating shards away from a node as soon as the node drops below the + high watermark. + + The decider handles multiple data paths by only accounting for relocating + shards that affect the appropriate data path. However, this mechanism does not + correctly account for _new_ relocating shards, which are unwittingly ignored. + This means that we may evict far too many shards from a node above the high + watermark, and may relocate far too many shards onto a node causing it to blow + right past the low watermark and potentially other watermarks too. + + There are in fact two distinct issues that this PR fixes. New incoming shards + have an unknown data path until the `ClusterInfoService` refreshes its + statistics. New outgoing shards have a known data path, but we fail to account + for the change of the corresponding `ShardRouting` from `STARTED` to + `RELOCATING`, meaning that we fail to find the correct data path and treat the + path as unknown here too. + + This PR also reworks the `MockDiskUsagesIT` test to avoid using fake data paths + for all shards. With the changes here, the data paths are handled in tests as + they are in production, except that their sizes are fake. + + Fixes #45177 + +commit b526309fbd435e18b1957c3d1830282f220ab7d8 +Author: Tanguy Leroux +Date: Thu Aug 29 13:15:48 2019 +0200 + + Replace MockAmazonS3 usage in S3BlobStoreRepositoryTests by a HTTP server (#46081) + + This commit removes the usage of MockAmazonS3 in S3BlobStoreRepositoryTests + and replaces it by a HttpServer that emulates the S3 service. This allows the + repository tests to use the real Amazon's S3 client under the hood in tests and will + allow to test the behavior of the snapshot/restore feature for S3 repositories by + simulating random server-side internal errors. + + The HTTP server used to emulate the S3 service is intentionally simple and minimal + to keep things understandable and maintainable. Testing full client options on the + server side (like authentication, chunked encoding etc) remains the responsibility + of the AmazonS3Fixture. + +commit 93ede78b661797e0e26c7dc0fa2cf2871408b8d5 +Author: István Zoltán Szabó +Date: Thu Aug 29 11:59:21 2019 +0200 + + Revert "[DOCS] Adds search-related query parameters to the common parameters. (#46057)" + + This reverts commit 95a50ae809cf62de45d6a69c622c21b01eef42aa. + +commit 4b086fbef24511eb03dde22db37d383396b083c7 +Author: István Zoltán Szabó +Date: Thu Aug 29 11:58:28 2019 +0200 + + Revert "[DOCS] Reformats URI search request (#45844)" + + This reverts commit 7f11c3240018b4005f918c4850cc38ef23d7aeb8. + +commit 95a50ae809cf62de45d6a69c622c21b01eef42aa +Author: István Zoltán Szabó +Date: Thu Aug 29 01:44:22 2019 +0200 + + [DOCS] Adds search-related query parameters to the common parameters. (#46057) + + @szabosteve Merging so I can make some additions. Will incorporate the comments from @jrodewig. + +commit fbe9e8a53062d250b1f66d29c7d18ebfafe9a153 +Author: Przemysław Witek +Date: Thu Aug 29 10:47:17 2019 +0200 + + Do not throw an exception if the process finished quickly but without any error. (#46073) (#46113) + +commit 3666bcfbd8a9819e225a62894ad00861f3b82615 +Author: Rory Hunter +Date: Thu Aug 29 09:45:51 2019 +0100 + + Handle multiple loopback addresses (#46061) + + AbstractSimpleTransportTestCase.testTransportProfilesWithPortAndHost + expects a host to only have a single IPv4 loopback address, which isn't + necessarily the case. Allow for >= 1 address. + + Backport of #45901. + +commit 867cfe0223189e6ec90efe5e265a92363dd8fb85 +Author: Costin Leau +Date: Thu Aug 29 11:13:18 2019 +0300 + + DOC: Update SQL docs for DbVis and Workbench/J (#45981) + + Refresh the setup for the new versions of DbVisualizer and SQL + Workbench/J which have Elasticsearch JDBC support out of the box. + + (cherry picked from commit 6d257194c1055d060505e0faaaa37b41e21699f5) + +commit 7f11c3240018b4005f918c4850cc38ef23d7aeb8 +Author: István Zoltán Szabó +Date: Thu Aug 29 10:04:49 2019 +0200 + + [DOCS] Reformats URI search request (#45844) + + * [DOCS] Reformats URI search request. + + Co-Authored-By: James Rodewig + + Co-Authored-By: debadair + +commit 0425f6a327c398ea81bf805302ca12251b6fd832 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Aug 29 07:51:46 2019 +0200 + + Docs _cat/health verification fix (#46064) + + The _cat/health call in getting-started assumes that the master task max + wait time is always 0 (-), however, the test could sometimes run into a + short wait time (like some ms). Fixed to allow this. + +commit 9bc4a241180693edef657471ed7f965c8ac9c183 +Author: Jason Tedor +Date: Wed Aug 28 22:16:15 2019 -0400 + + Handle delete document level failures (#46100) + + Today we assume that document failures can not occur for deletes. This + assumption is bogus, as they can fail for a variety of reasons such as + the Lucene index having reached the document limit. Because of this + assumption, we were asserting that such a document-level failure would + never happen. When this bogus assertion is violated, we fail the node, a + catastrophe. Instead, we need to treat this as a fatal engine exception. + +commit 70507e104195f1e47d85e59e50a32673d7e379a7 +Author: Tim Brooks +Date: Wed Aug 28 19:14:41 2019 -0600 + + Move netty numDirectArenas to jvm.options (#46104) + + We currently configure io.netty.allocator.numDirectArenas to be 0 in the + jvm erconomics class. This is a config that we always want to set, so it + makes sense to move it to jvm.options. + +commit 28006fe19ff1ff4e6e9c3b284518b4cae6137e50 +Author: Igor Motov +Date: Wed Aug 21 08:45:51 2019 -0400 + + Fix GeoIpProcessorFactoryTests on windows (#45668) + + Switches windows build to use geoip database loaded on heap instead + of memory mapping it. + + Closes #44552 + +commit 47bbd9d9a98651c9f26409bd0881d08e13cb486d +Author: Gordon Brown +Date: Wed Aug 28 14:50:22 2019 -0700 + + [7.x] Fix rollover alias in SLM history index template (#46001) + + This commit adds the `rollover_alias` setting required for ILM to work + correctly to the SLM history index template and adds assertions to the + SLM integration tests to ensure that it works correctly. + +commit a356bcff41d34055f294d43e2fff9515cc7bd7b1 +Author: Tal Levy +Date: Wed Aug 28 14:44:08 2019 -0700 + + Add Circle Processor (#43851) (#46097) + + add circle-processor that translates circles to polygons + +commit d94c4dcffbf8d77a9d58321efc6ca92f9526b4bb +Author: Julie Tibshirani +Date: Wed Aug 28 11:00:15 2019 -0700 + + Use float instead of double for query vectors. (#46004) + + Currently, when using script_score functions like cosineSimilarity, the query + vector is treated as an array of doubles. Since the stored document vectors use + floats, it seems like the least surprising behavior for the query vectors to + also be float arrays. + + In addition to improving consistency, this change may help with some + optimizations we have been considering around vector dot product. + +commit 1249e6ba5d12bf6cd2e8935a8e6a1fb87e83702a +Author: Jason Tedor +Date: Wed Aug 28 13:44:39 2019 -0400 + + Handle no-op document level failures (#46083) + + Today we assume that document failures can not occur for no-ops. This + assumption is bogus, as they can fail for a variety of reasons such as + the Lucene index having reached the document limit. Because of this + assumption, we were asserting that such a document-level failure would + never happen. When this bogus assertion is violated, we fail the node, a + catastrophe. Instead, we need to treat this as a fatal engine exception. + +commit 564b80303d91e1531a1ac370bae2e3fc2f1198e2 +Author: Ryan Ernst +Date: Wed Aug 28 10:53:38 2019 -0700 + + Fix rest-api-spec dep for external plugins (#45949) + + This commit fixes the maven coordinates for the rest-api-spec jar. It + was accidentally by #45107. + + closes #45891 + +commit f20969959fa9a7edc4feadb8f5167462bb1af598 +Author: Ryan Ernst +Date: Wed Aug 28 10:48:41 2019 -0700 + + Remove plugins dir reference from docs (#46047) + + While the plugin installation directory used to be settable, it has not + been so for several major versions. This commit removes a lingering + reference to the plugins directory in upgrade docs. + + closes #45889 + +commit 9e14ffa8be205d88775f9535c9a5f6c9521039ba +Author: Tanguy Leroux +Date: Wed Aug 28 16:13:31 2019 +0200 + + Few clean ups in ESBlobStoreRepositoryIntegTestCase (#46068) + +commit f50c7cf88b7fdd359e186bad8993cfe9bc95f012 +Author: Martijn van Groningen +Date: Wed Aug 28 16:16:11 2019 +0200 + + Add XContentType as parameter to HLRC ART#createServerTestInstance (#46036) + + Add XContentType as parameter to the + AbstractResponseTestCase#createServerTestInstance method. + + In the case a server side response class serializes xcontent as + bytes then the test needs to know what xcontent type was randomily selected. + + This change is needed in #45970 + +commit 9ac85a4a2bfe6166f36f9be5fc1bb81a4fcce3de +Author: Mark Tozzi +Date: Wed Aug 28 09:31:48 2019 -0400 + + Fix compilation in CumulativeCardinalityAggregatorTests + +commit 54a882ada9fe8b5769563998f787f1a70fa41388 +Author: James Rodewig +Date: Wed Aug 28 08:58:09 2019 -0400 + + [DOCS] Add index alias exists API docs (#46042) + +commit aec125faff502730d1c5b26046d05df9c115a560 +Author: Mark Tozzi +Date: Wed Aug 28 09:06:09 2019 -0400 + + Support Range Fields in Histogram and Date Histogram (#46012) + + Backport of 1a0dddf4ad24b3f2c751a1fe0e024fdbf8754f94 (AKA #445395) + + * Add support for a Range field ValuesSource, including decode logic for range doc values and exposing RangeType as a first class enum + * Provide hooks in ValuesSourceConfig for aggregations to control ValuesSource class selection on missing & script values + * Branch aggregator creation in Histogram and DateHistogram based on ValuesSource class, to enable specialization based on type. This is similar to how Terms aggregator works. + * Prioritize field type when available for selecting the ValuesSource class type to use for an aggregation + +commit f28644c4981058ab2fe0f3a4e6d6f30e11391913 +Author: James Rodewig +Date: Wed Aug 28 08:32:05 2019 -0400 + + [DOCS] Add upgrade support matrix (#45790) + +commit 25d64508f6b510facb33529185fa5a83636f2364 +Author: Dimitris Athanasiou +Date: Wed Aug 28 12:02:29 2019 +0300 + + [7.x][ML] Support boolean fields for DF analytics (#46037) (#46054) + + This commit adds support for `boolean` fields in data frame + analytics (and currently both outlier detection and regression). + The analytics process expects `boolean` fields to be encoded as + integers with 0 or 1 value. + +commit bb8fcb3cacf606f432a321d8d825db51dabdfa9a +Author: Dimitris Athanasiou +Date: Wed Aug 28 12:02:14 2019 +0300 + + [7.x][ML][HLRC] Add data frame analytics regression analysis (#46024) (#46053) + +commit 1157224a6b070ccd9854d5d5a355f4ef43f1c75d +Merge: b170b76670f 154d1dd962b +Author: Martijn van Groningen +Date: Wed Aug 28 10:14:07 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 154d1dd962b79821abf35c95051ed5888517f163 +Author: Jake Landis +Date: Tue Aug 27 16:57:20 2019 -0500 + + Watcher max_iterations with foreach action execution (#45715) (#46039) + + Prior to this commit the foreach action execution had a hard coded + limit to 100 iterations. This commit allows the max number of + iterations to be a configuration ('max_iterations') on the foreach + action. The default remains 100. + +commit 956df7be92168a13c4858fd746a2d0596dce655f +Author: Tim Brooks +Date: Tue Aug 27 15:28:04 2019 -0500 + + Reindex task state initialized before reindex (#46043) + + Currently the process to execute a reindex process is tightly coupled to + step of initializing the task state. This creates problems when this + process is asynchronous. It is possible that the task state has not been + initialized which prevents follow-up actions such as rethrottle. This + commit separates the task initialization so that it can be executed as a + first step in the persistent reindex process. + +commit 07f3ddb54949b5b63f91ff03f5dca26a90966abc +Author: Tim Brooks +Date: Tue Aug 27 12:28:37 2019 -0500 + + Extract reindexing logic from transport action (#46033) + + This commit extracts the reindexing logic from the transport action so + that it can be incorporated into the persistent reindex work without + requiring the usage of the client. + +commit ff1acf34893584328927ac018b7e3337dde4b827 +Author: James Rodewig +Date: Tue Aug 27 12:49:18 2019 -0400 + + [DOCS] Reformat update index settings API docs (#45931) + +commit a3d7547e10e281bdf52b87e75afec641218e957a +Author: James Rodewig +Date: Tue Aug 27 12:31:53 2019 -0400 + + [DOCS] Separate and reformat close index API docs (#45922) + +commit ad233e3e38cd83ca87f9df1c7ee84a2da5658c76 +Author: Tim Brooks +Date: Tue Aug 27 11:25:22 2019 -0500 + + Add test for CopyBytesSocketChannel (#46031) + + Currently we use a custom CopyBytesSocketChannel for interfacing with + netty. We have integration tests that use this channel, however we never + verify the read and write behavior in the face of potential partial + writes. This commit adds a test for this behavior. + +commit 8228a218b421a29b3e85f130a95cb2acfa89ff66 +Author: James Rodewig +Date: Tue Aug 27 11:17:51 2019 -0400 + + [DOCS] Reformat open index API docs (#45921) + +commit fdef293c812c61bccc8eedf7bd8360e001f5c0a7 +Author: Armin Braun +Date: Tue Aug 27 17:23:48 2019 +0200 + + Fix RegressionTests#fromXContent (#46029) + + * The `trainingPercent` must be between `1` and `100`, not `0` and `100` which is causing test failures + +commit 4b879848f06e026a727af944e16a63feb521a7bd +Author: Lisa Cawley +Date: Tue Aug 27 08:10:13 2019 -0700 + + [DOCS] Add 7.3.1 ml-cpp PRs to release notes (#46003) + +commit 300e717e421c30a629f2db028a44c6a6d89465f4 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Aug 27 14:17:15 2019 +0200 + + Disallow partial results when shard unavailable (#45739) + + Searching with `allowPartialSearchResults=false` could still return + partial search results during recovery. If a shard copy fails + with a "shard not available" exception, the failure would be ignored and + a partial result returned. The one case where this is known to happen + is when a shard copy is recovering when searching, since + `IllegalIndexShardStateException` is considered a "shard not available" + exception. + + Relates to #42612 + +commit 873ad3f942da229e859f8613150114857ea52d8b +Author: Dimitris Athanasiou +Date: Tue Aug 27 17:53:11 2019 +0300 + + [7.x][ML] Add option to regression to randomize training set (#45969) (#46017) + + Adds a parameter `training_percent` to regression. The default + value is `100`. When the parameter is set to a value less than `100`, + from the rows that can be used for training (ie. those that have a + value for the dependent variable) we randomly choose whether to actually + use for training. This enables splitting the data into a training set and + the rest, usually called testing, validation or holdout set, which allows + for validating the model on data that have not been used for training. + + Technically, the analytics process considers as training the data that + have a value for the dependent variable. Thus, when we decide a training + row is not going to be used for training, we simply clear the row's + dependent variable. + +commit 7b6246ec670040e8e5dfef05b33bb2b39f8a9357 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Aug 28 00:44:23 2019 +1000 + + Add `manage_own_api_key` cluster privilege (#45897) (#46023) + + The existing privilege model for API keys with privileges like + `manage_api_key`, `manage_security` etc. are too permissive and + we would want finer-grained control over the cluster privileges + for API keys. Previously APIs created would also need these + privileges to get its own information. + + This commit adds support for `manage_own_api_key` cluster privilege + which only allows api key cluster actions on API keys owned by the + currently authenticated user. Also adds support for retrieval of + the API key self-information when authenticating via API key + without the need for the additional API key privileges. + To support this privilege, we are introducing additional + authentication context along with the request context such that + it can be used to authorize cluster actions based on the current + user authentication. + + The API key get and invalidate APIs introduce an `owner` flag + that can be set to true if the API key request (Get or Invalidate) + is for the API keys owned by the currently authenticated user only. + In that case, `realm` and `username` cannot be set as they are + assumed to be the currently authenticated ones. + + The changes cover HLRC changes, documentation for the API changes. + + Closes #40031 + +commit dd6c13fdf9678211781cc5470b7a9987b760b641 +Author: Dimitris Athanasiou +Date: Tue Aug 27 15:48:59 2019 +0300 + + [ML] Add description to DF analytics (#45774) (#46019) + +commit 267183998ebc48b859f76d5bdfca65b569b553df +Author: Luca Cavanna +Date: Tue Aug 27 09:07:59 2019 +0200 + + [TEST] wait for http channels to be closed in ESIntegTestCase (#45977) + + We recently added a check to `ESIntegTestCase` in order to verify that + no http channels are being tracked when we close clusters and the + REST client. Close listeners though are invoked asynchronously, hence + this check may fail if we assert before the close listener that removes + the channel from the map is invoked. + + With this commit we add an `assertBusy` so we try and wait for the map + to be empty. + + Closes #45914 + Closes #45955 + +commit 1ebee5bf9b6e0a43eb8eebe8215b85a626534ae9 +Author: Albert Zaharovits +Date: Tue Aug 27 14:42:46 2019 +0300 + + PKI realm authentication delegation (#45906) + + This commit introduces PKI realm delegation. This feature + supports the PKI authentication feature in Kibana. + + In essence, this creates a new API endpoint which Kibana must + call to authenticate clients that use certificates in their TLS + connection to Kibana. The API call passes to Elasticsearch the client's + certificate chain. The response contains an access token to be further + used to authenticate as the client. The client's certificates are validated + by the PKI realms that have been explicitly configured to permit + certificates from the proxy (Kibana). The user calling the delegation + API must have the delegate_pki privilege. + + Closes #34396 + +commit b249e25bb4fcd9e2ebd65f3b2135984f8aecd3e7 +Author: Ioannis Kakavas +Date: Tue Aug 27 13:00:33 2019 +0300 + + Partly revert globalInfo.ready check (#45960) + + This check was introduced in #41392 but had the unwanted side-effect + that the keystore settings in such blocks would note be added in the + node's keystore. Given that we have a mid-term plan for FIPS testing + that would made such checks unnecessary, and that the conditional + in these two cases is not really that important, this change removes + this conditional logic so that full-cluster-restart and rolling + upgrade tests will run with PEM files for key/certificate material + no matter if we're in a FIPS JVM or not. + + Resolves: #45475 + +commit cf34ff62ad3ed38a55f40d1b649f1bc12b1e8b00 +Author: debadair +Date: Mon Aug 26 13:58:41 2019 -0700 + + [DOCS] Streamline GS search topic. (#45941) + + * Streamline GS search topic. + + * Added missing comma. + + * Update docs/reference/getting-started.asciidoc + + Co-Authored-By: István Zoltán Szabó + +commit 948b03856b3bb657d824149e53e2908b532ad56c +Author: debadair +Date: Mon Aug 26 18:24:05 2019 -0700 + + [DOCS] Backporting GS search & aggs updates. (#46008) + + * [DOCS] Streamlined GS aggs section. (#45951) + + * [DOCS] Streamlined GS aggs section. + + * Update docs/reference/getting-started.asciidoc + + Co-Authored-By: James Rodewig + + * [DOCS] Fix typo. (#46006) + +commit d50d700f148b9d31a4c590ef751a85cc6e09250d +Author: Ryan Ernst +Date: Mon Aug 26 16:34:32 2019 -0700 + + Don't use assemble task on root project (#45999) + + The root project uses the base plugin to get a clean task, but does not + actually need the assemble task. This commit changes the root project to + use the lifecycle-base plugin, which while still creating the assemble + task, won't add any dependencies to it. + +commit 146e23a8a90410978c9c9033e943ab26c1e9244e +Author: Nhat Nguyen +Date: Mon Aug 26 09:08:22 2019 -0400 + + Relax translog assertion in testRestoreLocalHistoryFromTranslog (#45943) + + Since #45473, we trim translog below the local checkpoint of the safe + commit immediately if soft-deletes enabled. In + testRestoreLocalHistoryFromTranslog, we should have a safe commit after + recoverFromTranslog is called; then we will trim translog files which + contain only operations that are at most the global checkpoint. + + With this change, we relax the assertion to ensure that we don't put + operations to translog while recovering history from the local translog. + +commit c66bae39c34faf4a2228f332a8a3650d91dcbfeb +Author: Nhat Nguyen +Date: Fri Aug 23 11:09:49 2019 -0400 + + Update translog checkpoint after marking ops as persisted (#45634) + + If two translog syncs happen concurrently, then one can return before + its operations are marked as persisted. In general, this should not be + an issue; however, peer recoveries currently rely on this assumption. + + Closes #29161 + +commit f2e8b17696862f4d27d951b70f0c4431c8395de5 +Author: Nhat Nguyen +Date: Mon Aug 26 17:16:29 2019 -0400 + + Do not create engine under IndexShard#mutex (#45263) + + Today we create new engines under IndexShard#mutex. This is not ideal + because it can block the cluster state updates which also execute under + the same mutex. We can avoid this problem by creating new engines under + a separate mutex. + + Closes #43699 + +commit 1b4d5b37c5f67ffa3565a0003dfebe2c6f68df70 +Author: Jake Landis +Date: Thu Aug 22 13:29:39 2019 -0500 + + add release notes for 7.3.1 (#45871) + + Co-Authored-By: James Rodewig + +commit b3f76d6f22a400266fd16854450b6d7a1c999f3d +Author: Nik Everett +Date: Mon Aug 26 16:38:02 2019 -0400 + + Build: Support `console-result` language (#45937) (#45976) + + This adds support for verifying that snippets with the `console-result` + language are valid json. It also switches the response snippets on the + `docs/get` page from `js` to `console-result` which will allow clients + to provide "alternatives" for them like they can now do with + `// CONSOLE` snippets. + +commit 3d64605075f03239a55b366d17794b38d9f6f835 +Author: Jason Tedor +Date: Mon Aug 26 16:10:25 2019 -0400 + + Remove node settings from blob store repositories (#45991) + + This commit starts from the simple premise that the use of node settings + in blob store repositories is a mistake. Here we see that the node + settings are used to get default settings for store and restore throttle + rates. Yet, since there are not any node settings registered to this + effect, there can never be a default setting to fall back to there, and + so we always end up falling back to the default rate. Since this was the + only use of node settings in blob store repository, we move them. From + this, several places fall out where we were chaining settings through + only to get them to the blob store repository, so we clean these up as + well. That leaves us with the changeset in this commit. + +commit 943a016bb20c99bf3a45b1c1804abbb29c45bb53 +Author: Zachary Tong +Date: Mon Aug 26 16:19:55 2019 -0400 + + Add Cumulative Cardinality agg (and Data Science plugin) (#45990) + + This adds a pipeline aggregation that calculates the cumulative + cardinality of a field. It does this by iteratively merging in the + HLL sketch from consecutive buckets and emitting the cardinality up + to that point. + + This is useful for things like finding the total "new" users that have + visited a website (as opposed to "repeat" visitors). + + This is a Basic+ aggregation and adds a new Data Science plugin + to house it and future advanced analytics/data science aggregations. + +commit a3a4ae0ac25b9ec693a37c5198ae8c04acd2e48b +Author: Benjamin Trent +Date: Mon Aug 26 14:18:17 2019 -0500 + + [ML] fixing bug where analytics process starts with 0 rows (#45879) (#45988) + + The native process requires that there be a non-zero number of rows to analyze. If the flag --rows 0 is passed to the executable, it throws and does not start. + + When building the configuration for the process we should not start the native process if there are no rows. + + Adding some logging to indicate what is occurring. + +commit d64018f8e1615d79763f39f2d48af6105398dc1a +Author: Benjamin Trent +Date: Mon Aug 26 14:18:00 2019 -0500 + + [ML] add supported types to no fields error message (#45926) (#45987) + + * [ML] add supported types to no fields error message + + * adding supported types to logger debug + +commit 5535ff0a441c7c73d3118e9fd512234a3121f62b +Author: James Baiera +Date: Mon Aug 26 14:33:33 2019 -0400 + + Fix IngestService to respect original document content type (#45799) (#45984) + + Backport of #45799 + + This PR modifies the logic in IngestService to preserve the original content type + on the IndexRequest, such that when a document with a content type like SMILE + is submitted to a pipeline, the resulting document that is persisted will remain in + the original content type (SMILE in this case). + +commit ff7fd9b9e248cfaf99288be2cbba801404f39ea1 +Author: William Brafford +Date: Mon Aug 26 11:44:31 2019 -0400 + + Pass COMPUTERNAME env var to elasticsearch.bat (#45763) + + * Pass COMPUTERNAME env var to elasticsearch.bat + + When we run bin/elasticsearch with bash, we get a $HOSTNAME builtin that + contains the hostname of the machine the script is running on. When + there's no provided nodename, Elasticsearch uses the HOSTNAME to create + a nodename. On Windows, Powershell provides a $COMPUTERNAME variable for + the same purpose. CMD.EXE provides the same thing, except it's called + %COMPUTERNAME%. bin/elasticsearch.bat sets $HOSTNAME to the value of + $COMPUTERNAME. However, when testclusters invokes bin/elasticsearch.bat, + the COMPUTERNAME variable doesn't get passed in, leaving HOSTNAME null + and breaking an integration test on Windows. + + This commit sets COMPUTERNAME in the environment so that our tests get + the value that Elasticsearch would have when bin/elasticsearch.bat is + invoked from the shell. + + * Add null check to protect in non-Windows case + + What good is it a developer to gain the whole Windows if they forfeit + their Unix? The value that fixes things on Windows is null on + Linux/Darwin, so let's null-check it. + + * Override system hostnames for testclusters + + Rather than relying on variable system behavior, let's just override + HOSTNAME and COMPUTERNAME and test for correct values in the integration + test that was originally failing. + + * Rename constants for clarity + + Since we are setting HOSTNAME and COMPUTERNAME regardless of whether the + tests are running on Windows or Linux, we shouldn't imply that constants + are only used in one case or the other. + +commit db386617c7455eb6c70611059c980a33e0d8ecfe +Author: James Rodewig +Date: Mon Aug 26 11:13:18 2019 -0400 + + [DOCS] Correct `IIF` conditional section title (#45979) + +commit 5fbb57218354c88fed4dbb124379edd10b86324c +Author: James Rodewig +Date: Mon Aug 26 10:30:18 2019 -0400 + + [DOCS] Reformat get field mapping API docs (#45700) + +commit af2bd75def94cd15775cf6f661ff2151cb60abb2 +Author: Armin Braun +Date: Mon Aug 26 16:20:58 2019 +0200 + + Fix Broken HTTP Request Breaking Channel Closing (#45958) (#45973) + + This is essentially the same issue fixed in #43362 but for http request + version instead of the request method. We have to deal with the + case of not being able to parse the request version, otherwise + channel closing fails. + + Fixes #43850 + +commit d78bc487b4d2fdf92fc27cc61a8ada416e64c9a0 +Author: James Rodewig +Date: Mon Aug 26 10:07:11 2019 -0400 + + [DOCS] Reformat indices exists API docs (#45918) + +commit 5a17987e19e0aa015c6057ecc810f675dedc9b70 +Author: Armin Braun +Date: Mon Aug 26 15:59:02 2019 +0200 + + Fix SnapshotStatusApisIT (#45929) (#45971) + + The snapshot status when blocking can still be INIT in rare cases when + the new cluster state that has the snapshot in `STARTED` hasn't yet + become visible. + Fixes #45917 + +commit 9672ff388a0a1705358fb1da1567738477fdf678 +Author: Jason Tedor +Date: Mon Aug 26 09:37:25 2019 -0400 + + Consider artifact repositories backed by S3 secure (#45950) + + Since credentials are required to access such a repository, and these + repositories are accessed over an encrypted protocol (https), this + commit adds support to consider S3-backed artifact repositories as + secure. Additionally, we add tests for this functionality. + +commit 767f648f8ebad21e79bb7f0ab62b62f145228b38 +Author: Jake Landis +Date: Mon Aug 26 08:35:33 2019 -0500 + + Watcher add email warning if CSV attachment contains formulas (#44460) (#45557) + + * Watcher add email warning if CSV attachment contains formulas (#44460) + + This commit introduces a Warning message to the emails generated by + Watcher's reporting action. This change complements Kibana's CSV + formula notifications (see elastic/kibana#37930). + + This is implemented by reading a header (kbn-csv-contains-formulas) + provided by Kibana to notify to attach the Warning to the email. + The wording of the warning is borrowed from Kibana's UI and may + be overridden by a dynamic setting + xpack.notification.reporting.warning.kbn-csv-contains-formulas.text. + This warning is enabled by default, but may be disabled via a + dynamic setting xpack.notification.reporting.warning.enabled. + +commit f2241a152fb4e0935cf61f98fd0f0742973f6818 +Author: Jake Landis +Date: Mon Aug 26 08:34:54 2019 -0500 + + watcher tests - increase stop timeout to 60s (#45679) (#45934) + + As of #43939 Watcher tests now correctly block until all Watch executions + kicked off by that test are finished. Prior we allowed tests to finish with + outstanding watch executions. It was known that this would increase the + time needed to finish a test. However, running the tests on CI can be slow + and on at least 1 occasion it took 60s to actually finish. + + This PR simply increases the max allowable timeout for Watcher tests + to clean up after themselves. + +commit a3d918bddb06038e28b353e7831f358ae248e73f +Author: Tanguy Leroux +Date: Mon Aug 26 14:53:31 2019 +0200 + + Refactor RepositoryCredentialsTests (#45919) + + This commit refactors the S3 credentials tests in + RepositoryCredentialsTests so that it now uses a single + node (ESSingleNodeTestCase) to test how secure/insecure + credentials are overriding each other. Using a single node + makes it much easier to understand what each test is actually + testing and IMO better reflect how things are initialized. + + It also allows to fold into this class the test + testInsecureRepositoryCredentials which was wrongly located + in S3BlobStoreRepositoryTests. By moving this test away, the + S3BlobStoreRepositoryTests class does not need the + allow_insecure_settings option anymore and thus can be + executed as part of the usual gradle test task. + +commit d96469ddff3ad97de5be2e51267f853322ef953c +Author: Andrey Ershov +Date: Mon Aug 26 15:06:29 2019 +0200 + + Better logging for TLS message on non-secure transport channel (#45835) + + This commit enhances logging for 2 cases: + + 1. If non-TLS enabled node receives transport message from TLS enabled + node on transport port. + 2. If non-TLS enabled node receives HTTPs request on transport port. + + (cherry picked from commit 4f52ebd32eb58526b4c8022f8863210bf88fc9be) + +commit 1b90019599c10a24f5481ff31e69cc4ec5e69a52 +Author: James Rodewig +Date: Mon Aug 26 08:57:00 2019 -0400 + + [DOCS] Reformat get settings API docs (#45924) + +commit db1ac81d63f94609598d7793413fade323d459d9 +Author: James Rodewig +Date: Mon Aug 26 08:31:31 2019 -0400 + + [DOCS] Reformat create index API docs (#45749) + +commit 479ab9b8db10351740d11f7c6f1bda819c193489 +Author: Andrey Ershov +Date: Mon Aug 26 12:17:28 2019 +0200 + + Fix plaintext on TLS port logging (#45852) + + Today if non-TLS record is received on TLS port generic exception will + be logged with the stack-trace. + SSLExceptionHelper.isNotSslRecordException method does not work because + it's assuming that NonSslRecordException would be top-level. + This commit addresses the issue and the log would be more concise. + + (cherry picked from commit 6b83527bf0c23d4d5b97fab7f290c43432945d4f) + +commit b170b76670f63baca578ee3c7ee3eaabeeb5d82e +Author: Martijn van Groningen +Date: Mon Aug 26 09:54:25 2019 +0200 + + Add HLRC support for delete policy api (#45833) + + This PR also adds HLRC docs. + + Relates to #32789 + +commit 2bee27dd54bd8e101b5613d4a45861c5f433ee9a +Author: Ioannis Kakavas +Date: Sun Aug 25 19:36:41 2019 +0300 + + Allow Transport Actions to indicate authN realm (#45946) + + This commit allows the Transport Actions for the SSO realms to + indicate the realm that should be used to authenticate the + constructed AuthenticationToken. This is useful in the case that + many authentication realms of the same type have been configured + and where the caller of the API(Kibana or a custom web app) already + know which realm should be used so there is no need to iterate all + the realms of the same type. + The realm parameter is added in the relevant REST APIs as optional + so as not to introduce any breaking change. + +commit 040a810b3c11cc95a2e921ddc6f3dbc8c16f6e46 +Author: Jason Tedor +Date: Sat Aug 24 17:19:20 2019 -0400 + + Add deprecation check for pidfile setting (#45939) + + The pidfile setting is deprecated. This commit adds a deprecation check + for usage of this setting. + +commit 599bf2d68bbc7fd344e075021c1650bc10b227cd +Author: Jason Tedor +Date: Fri Aug 23 21:27:55 2019 -0400 + + Deprecate the pidfile setting (#45938) + + This commit deprecates the pidfile setting in favor of node.pidfile. + +commit 43ca652d11ffe71d4d203a2ff62b54cb64455cc0 +Author: Jason Tedor +Date: Fri Aug 23 20:16:40 2019 -0400 + + Add deprecation check for processors (#45925) + + The processors setting is deprecated. This commit adds a deprecation + check for the use of the processors setting. + +commit 377ff7e9ce73219db79f6acebc715912a47851a5 +Author: Jason Tedor +Date: Fri Aug 23 18:36:31 2019 -0400 + + Fix 7.4 migration docs + + This page was using the wrong formatting markup. + +commit a9f5bb91c291a9609348dfc8ac34e9a761b3fe63 +Author: Jason Tedor +Date: Fri Aug 23 18:09:32 2019 -0400 + + Include 7.4 migration docs in 7.x branch + + This commit adds the 7.4 migration docs to the 7.x branch. + +commit 6b116a48f3f2c0dafda24de04c8e5378c339971d +Author: Jason Tedor +Date: Fri Aug 23 17:38:15 2019 -0400 + + Skip feature aware check on JDK 14 (#45928) + + ASM can not currently handle classes compiled with JDK 14. This commit + skips these checks on JDK 14, for now. + +commit 00f931a59aa35b5cd91f0440fb79ccca7811565e +Author: Jason Tedor +Date: Fri Aug 23 17:11:04 2019 -0400 + + Add deprecation note on processors setting + + This commit adds a note to the docs that the processors setting has been + deprecated in favor of node.processors. + +commit a82d24b3ce05592c7dec76f2b1df694ce3cb3e38 +Author: Michael Basnight +Date: Fri Aug 23 14:34:13 2019 -0500 + + Remove enrich indices on delete policy (#45870) + + When a policy is deleted, the enrich indices that are backing the policy + alias should also be deleted. This commit does that work and cleans up + the transport action a bit so that the lock release is easier to see, as + well as to ensure that any action carried out, regardless of exception, + unlocks the policy. + +commit be554fe5f03f95f0df53e1a783363594cf352649 +Author: Dimitris Athanasiou +Date: Fri Aug 23 23:04:39 2019 +0300 + + [7.x][ML] Improve progress reportings for DF analytics (#45856) (#45910) + + Previously, the stats API reports a progress percentage + for DF analytics tasks that are running and are in the + `reindexing` or `analyzing` state. + + This means that when the task is `stopped` there is no progress + reported. Thus, one cannot distinguish between a task that never + run to one that completed. + + In addition, there are blind spots in the progress reporting. + In particular, we do not account for when data is loaded into the + process. We also do not account for when results are written. + + This commit addresses the above issues. It changes progress + to being a list of objects, each one describing the phase + and its progress as a percentage. We currently have 4 phases: + reindexing, loading_data, analyzing, writing_results. + + When the task stops, progress is persisted as a document in the + state index. The stats API now reports progress from in-memory + if the task is running, or returns the persisted document + (if there is one). + +commit b756e1b9be92123528dc6e46a3ef7ee7743e9a08 +Author: Benjamin Trent +Date: Fri Aug 23 13:53:02 2019 -0500 + + [ML][Transforms] adjusting when and what to audit (#45876) (#45916) + + * [ML][Transforms] adjusting when and what to audit + + * Update DataFrameTransformTask.java + + * removing unnecessary audit message + +commit f3825767f44745ec456a7656a5d736a672ceeb7e +Author: James Rodewig +Date: Fri Aug 23 11:54:40 2019 -0400 + + [DOCS] Relocate Ingest API docs to REST API section (#45812) + +commit 8cac0638a14cc414dd5650029fe72f73997e1e05 +Author: James Rodewig +Date: Fri Aug 23 11:29:02 2019 -0400 + + [DOCS] Reformat delete index API docs (#45755) + +commit 7367855c99ac770fa19fa0877081fb0c0d04448c +Author: James Rodewig +Date: Fri Aug 23 11:14:51 2019 -0400 + + [DOCS] Reformat get index API docs (#45758) + +commit 45ad01ab1c0c5df409df5c45d7ef99ec64c80b66 +Author: Jack Conradson +Date: Fri Aug 23 08:07:06 2019 -0700 + + Fix bugs in Painless SCatch node (#45880) + + This fixes two bugs: + - A recently introduced bug where an NPE will be thrown if a catch block is + empty. + - A long-time bug where an NPE will be thrown if multiple catch blocks in a + row are empty for the same try block. + +commit ceb8b9bbeee7eebc3c4c85a45336cfae57a0d420 +Author: James Rodewig +Date: Fri Aug 23 10:57:20 2019 -0400 + + Change `{var}` convention to `` (#45904) + +commit 94c2de65b9137ba8ee74f24e975caa5d0c192822 +Author: Benjamin Trent +Date: Fri Aug 23 09:38:52 2019 -0500 + + [ML][Transforms] fix doSaveState check (#45882) (#45902) + + * [ML][Transforms] fix doSaveState check + + * removing unnecessary log statement + +commit 3bc1494d3860ccc055b4cc36861d60a5c2ddde69 +Author: Mayya Sharipova +Date: Fri Aug 23 10:30:36 2019 -0400 + + Correct warning testScalingThreadPoolConfiguration + + Correct expected warning + + Closes #45907 + +commit 01acabb64c089300e8779f5362a5560ba7d0151b +Author: James Rodewig +Date: Fri Aug 23 08:13:27 2019 -0400 + + [DOCS] Reformat put mapping API docs (#45709) + +commit 46d9a575dbece9117cb1e752e08b31807e3fe41e +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Aug 23 14:25:26 2019 +0200 + + Fix RemoteClusterConnection close race (#45898) + + Closing a `RemoteClusterConnection` concurrently with trying to connect + could result in double invoking the listener. + + This fixes + RemoteClusterConnectionTest#testCloseWhileConcurrentlyConnecting + + Closes #45845 + +commit a38e6850a5024bcc6983191b44abcfbe01463d3a +Author: Martijn van Groningen +Date: Fri Aug 23 13:51:00 2019 +0200 + + fixed errors after cherry-picking 2 commits + +commit 6067065ed66511331aae50dbfe5d6e78633ec2a2 +Author: Martijn van Groningen +Date: Fri Aug 23 13:38:27 2019 +0200 + + Decouple enrich processor factory from enrich policy (#45826) + + This commit changes the enrich processor factory to read the required + configuration from the current enrich index (from meta mapping field) + in order to create the processor. + + Before this change the required config was read from the enrich policy + in the cluster state. Enrich policies are going to be stored in an + index (instead of the cluster state). In a processor factory there isn't + a way to load something from an index, so with this change we read + the required config / info from the enrich index (which is derived + from the enrich policy), which then allows us to move enrich policies + to an index. + + With this change it is required to execute a policy before creating a + pipeline. Otherwise there is no enrich index and then there is no way + to validate that a policy exist or retrieve its type and match field. + + Relates to #32789 + +commit 8e66df99257786c989d547672fbccc530a9035e6 +Author: Tanguy Leroux +Date: Fri Aug 23 13:40:39 2019 +0200 + + Move testRetentionLeasesClearedOnRestore (#45896) + +commit cb42e19a321403b1b0327fbacea18eb62c7f4bed +Author: Martijn van Groningen +Date: Fri Aug 23 13:38:12 2019 +0200 + + Change how type is stored in an enrich policy. (#45789) + + A policy type controls how the enrich index is created and + the query executed against the match field. Currently there + is a single policy type (`exact_match`). In the near future + more policy types will be added and different policy may have + different configuration options. + + For this reason type should be a json object instead of a string field: + + ``` + { + "exact_match": { + ... + } + } + ``` + + instead of: + + ``` + { + "type": "exact_match", + ... + } + ``` + + This will make streaming parsing of enrich policies easier as in the + new format, the parsing code can know ahead what configuration fields + to expect. In the latter format that is not possible if the type field + appears not as the first field. + + Relates to #32789 + +commit aee92d573cb6a7a32a402a1f55e7d4a5da968625 +Author: Tanguy Leroux +Date: Fri Aug 23 13:38:52 2019 +0200 + + Allow partial request body reads in AWS S3 retries tests (#45847) + + This commit changes the tests added in #45383 so that the fixture that + emulates the S3 service now sometimes consumes all the request body + before sending an error, sometimes consumes only a part of the request + body and sometimes consumes nothing. The idea here is to beef up a bit + the tests that writes blob because the client's retry logic relies on + marking and resetting the blob's input stream. + + This pull request also changes the testWriteBlobWithRetries() so that it + (rarely) tests with a large blob (up to 1mb), which is more than the client's + default read limit on input streams (131Kb). + + Finally, it optimizes the ZeroInputStream so that it is a bit more effective + (now works using an internal buffer and System.arraycopy() primitives). + +commit 6e696296fedf0896af0b6afbb165cb771177c48d +Author: István Zoltán Szabó +Date: Thu Aug 22 15:04:20 2019 +0200 + + [DOCS] Reformats search API (#45786) + + Co-Authored-By: James Rodewig + +commit 837cfa2640d583f0700063a600c47147d5166271 +Merge: 33972423e9b 217e41ab6c9 +Author: Martijn van Groningen +Date: Fri Aug 23 11:22:27 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit ecafe4f4ad3fccdbad69c0857666b737c956715c +Author: Alexander Reelsen +Date: Fri Aug 23 10:39:03 2019 +0200 + + Update joda to 2.10.3 (#45495) + +commit 217e41ab6c9f95299436e296c1718fee4585b899 +Author: markharwood +Date: Fri Aug 23 09:22:17 2019 +0100 + + Search - added HLRC support for PinnedQueryBuilder (#45779) (#45853) + + Added HLRC support for PinnedQueryBuilder + + Related #44074 + +commit 85d55e30d0bb83ef705239a3ff4c1f1aad8a296c +Author: Przemysław Witek +Date: Fri Aug 23 07:03:09 2019 +0200 + + Add test that proves _timing_stats document is deleted when the job is deleted (#45840) (#45854) + +commit 2ed19b2c817fb4068888fb6246d3fda08d0460e4 +Author: Przemysław Witek +Date: Fri Aug 23 07:02:50 2019 +0200 + + Put error message from inside the process into the exception that is thrown when the process doesn't start correctly. (#45846) (#45875) + +commit ba6d72ea9f773ccf3c673ba2c32f8c54e656c0ae +Author: Armin Braun +Date: Fri Aug 23 06:17:50 2019 +0200 + + Fix TransportSnapshotsStatusAction ThreadPool Use (#45824) (#45883) + + In case of an in-progress snapshot this endpoint was broken because + it tried to execute repository operations in the callback on a + transport thread which is not allowed (only generic or snapshot + pool are allowed here). + +commit f94e4a9151a1b747dc79509f00762b99a98ae6b2 +Author: Tim Vernum +Date: Fri Aug 23 12:41:37 2019 +1000 + + Set security index refresh interval to 1s (#45888) + + The security indices were being created without specifying the + refresh interval, which means it would inherit a value from any + templates that exists. + However, certain security functionality depends on being able to + wait_for refresh, and causes errors (e.g. in Kibana) if that time + exceeds 30s. + + This commit changes the security indices configuration to always be + created with a 1s refresh interval. This prevents any templates from + inadvertantly interfering with the proper functioning of security. + It is possible for an administrator to explicitly change the refresh + interval after the indices have been created. + + Backport of: #45434 + +commit 243f054b0b6b066a2e3f315ed6c406888c508ec9 +Author: Jason Tedor +Date: Thu Aug 22 22:20:56 2019 -0400 + + Remove redundant Java check from Sys V init (#45793) + + In the Sys V init scripts, we check for Java. This is not needed, since + the same check happens in elasticsearch-env when starting up. Having + this duplicate check has bitten us in the past, where we made a change + to the logic in elasticsearch-env, but missed updating it here. Since + there is no need for this duplicate check, we remove it from the Sys V + init scripts. + +commit de6b6fd338dcc0fc6fd4888aaac0c18f360b036b +Author: Jason Tedor +Date: Thu Aug 22 22:18:37 2019 -0400 + + Add node.processors setting in favor of processors (#45885) + + This commit namespaces the existing processors setting under the "node" + namespace. In doing so, we deprecate the existing processors setting in + favor of node.processors. + +commit bdfd90560f6a3f966ca380ff834afebef3a49d57 +Author: Nathan Howard +Date: Thu Aug 22 18:22:54 2019 -0700 + + Adding a warning to from-size.asciidoc + + Customers occasionally discover a known behavior in Elasticsearch's pagination that does not appear to be documented. This warning is intended to educate customers of this behavior while still highlighting alternative solutions. + +commit 0d208dedf62cd4638b1cab3ce58be65b0465ef13 +Author: debadair +Date: Thu Aug 22 17:13:07 2019 -0700 + + [DOCS] Add anchor to version types list. (#45886) + +commit 029725fc351cbbe0ae707ff0be281b857b876e84 +Author: Tim Vernum +Date: Fri Aug 23 10:13:51 2019 +1000 + + Add SSL/TLS settings for watcher email (#45836) + + This change adds a new SSL context + + xpack.notification.email.ssl.* + + that supports the standard SSL configuration settings (truststore, + verification_mode, etc). This SSL context is used when configuring + outbound SMTP properties for watcher email notifications. + + Backport of: #45272 + +commit e38289b94c85123bcef82833e0f3cb9fdaba8550 +Author: Jason Tedor +Date: Thu Aug 22 19:34:17 2019 -0400 + + Enable testing against JDK 14 (#45178) + + This commit enables testing against JDK 14. + +commit c6f931cb220d7665a0d0614f010e380bc21b9e78 +Author: debadair +Date: Tue Aug 13 09:54:04 2019 -0700 + + [DOCS] Reformat doc delete API reference. (#45422) + + * [DOCS] Reformat doc delete API reference. + +commit 7bf96519224cc5b839c087499a106558cb7b12c1 +Author: debadair +Date: Tue Aug 13 09:53:39 2019 -0700 + + [DOCS] Reformat update API reference. (#45423) + + * [DOCS] Reformat update API reference. + +commit 4ffb48d4b29e08aacf7b1ca4bf291969256a04c8 +Author: debadair +Date: Tue Aug 13 09:52:56 2019 -0700 + + [DOCS] Reformat get API. (#45416) + + * [DOCS] Reformat get API. + +commit e9e9526192ff667d785e3b232575112a4dc53285 +Author: debadair +Date: Mon Aug 12 14:50:15 2019 -0700 + + [DOCS] Reformat index API. (#45415) + + * [DOCS] Reformat index API. + + * Incorporated review feedback. + +commit 3393f9599ea452e7425535e6cf339d4c067b2aed +Author: Nhat Nguyen +Date: Thu Aug 22 16:40:06 2019 -0400 + + Ignore translog retention policy if soft-deletes enabled (#45473) + + Since #45136, we use soft-deletes instead of translog in peer recovery. + There's no need to retain extra translog to increase a chance of + operation-based recoveries. This commit ignores the translog retention + policy if soft-deletes is enabled so we can discard translog more + quickly. + + Backport of #45473 + Relates #45136 + +commit 72c6302d12b0ad9dfc598815c570bd237a4ac625 +Author: dengweisysu +Date: Fri Aug 23 04:16:13 2019 +0800 + + Fsync translog without writeLock before rolling (#45765) + + Today, when rolling a new translog generation, we block all write + threads until a new generation is created. This choice is perfectly + fine except in a highly concurrent environment with the translog + async setting. We can reduce the blocking time by pre-sync the + current generation without writeLock before rolling. The new step + would fsync most of the data of the current generation without + blocking write threads. + + Close #45371 + +commit 8e3c54fff75420637e5605f9f3e8bb8446dc42da +Author: Benjamin Trent +Date: Thu Aug 22 15:15:41 2019 -0500 + + [7.x] [ML] Adding data frame analytics stats to _usage API (#45820) (#45872) + + * [ML] Adding data frame analytics stats to _usage API (#45820) + + * [ML] Adding data frame analytics stats to _usage API + + * making the size of analytics stats 10k + + * adjusting backport + +commit af2df9f221dba2ec6f30a2ab2d9fade7587b7587 +Author: James Rodewig +Date: Thu Aug 22 15:29:14 2019 -0400 + + [DOCS] Reformat match query (#45152) + +commit 4afa413a01d8a08e51097747dd318f9d9e9d5928 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Aug 22 22:02:49 2019 +0200 + + Fix update-by-query script examples (#43907) + + Two examples had swapped the order of lang and code when creating a + script. + + Relates #43884 + +commit 3af36008bd1d028d32a7f6a925ed4a16d57890f6 +Author: Jake Landis +Date: Thu Aug 22 14:20:34 2019 -0500 + + update bwcVersions + +commit f82c0f56a6b100433cac2fa28ff7765f65de54f5 +Author: William Brafford +Date: Thu Aug 22 15:00:43 2019 -0400 + + Mute flaky RemoteClusterConnection test (#45850) + +commit dff3e636c2d6a32479269d9f18caf89d4efc54a1 +Author: Benjamin Trent +Date: Thu Aug 22 13:15:07 2019 -0500 + + [ML][Transforms] unifying logging, adding some more logging (#45788) (#45859) + + * [ML][Transforms] unifying logging, adding some more logging + + * using parameterizedMessage instead of string concat + + * fixing bracket closure + +commit c60399c77f2f8a5c15100978e3a0ecc0b0c73237 +Author: Jake Landis +Date: Thu Aug 22 12:24:19 2019 -0500 + + introduce 7.3.2 version to 7.x (#45864) + +commit e50a78cf503655ea97a52d779192fee2afd3a026 +Author: Benjamin Trent +Date: Thu Aug 22 11:46:30 2019 -0500 + + [ML-DataFrame] version data frame transform internal index (#45375) (#45837) + + Adds index versioning for the internal data frame transform index. Allows for new indices to be created and referenced, `GET` requests now query over the index pattern and takes the latest doc (based on INDEX name). + +commit 1dab73929f7d6b9a3105dfd7c77d57de66a95d8e +Author: Jake Landis +Date: Thu Aug 22 10:54:29 2019 -0500 + + Watcher add stopped listener (#43939) (#45670) + + When Watcher is stopped and there are still outstanding watches running + Watcher will report it self as stopped. In normal cases, this is not problematic. + + However, for integration tests Watcher is started and stopped between + each test to help ensure a clean slate for each test. The tests are blocking + only on the stopped state and make an implicit assumption that all watches are + finished if the Watcher is stopped. This is an incorrect assumption since + Stopped really means, "I will not accept any more watches". This can lead to + un-predictable behavior in the tests such as message : "Watch is already queued + in thread pool" and state: "not_executed_already_queued". + This can also change the .watcher-history if watches linger between tests. + + This commit changes the semantics of a manual stopping watcher to now mean: + "I will not accept any more watches AND all running watches are complete". + There is now an intermediary step "Stopping" and callback to allow transition + to a "Stopped" state when all Watches have completed. + + Additionally since this impacts how long the tests will block waiting for a + "Stopped" state, the timeout has been increased. + + Related: #42409 + +commit bfddaaa2aefe52418a316e2a33c9db9b1399eacd +Author: Armin Braun +Date: Thu Aug 22 17:19:51 2019 +0200 + + Acknowledge Indices Were Wiped Successfully in REST Tests (#45832) (#45842) + + In internal test clusters tests we check that wiping all indices was acknowledged + but in REST tests we didn't. + This aligns the behavior in both kinds of tests. + Relates #45605 which might be caused by unacked deletes that were just slow. + +commit a1b88ca0098c548ef68b684e6be0169bd4d89517 +Author: Jack Conradson +Date: Thu Aug 22 06:46:23 2019 -0700 + + Move regex error to node (#45813) + +commit 33d4801213c603cb349ab184e6ab7ed14d9eb4c7 +Author: James Rodewig +Date: Thu Aug 22 09:49:15 2019 -0400 + + Revert "[DOCS] Reformats search API (#45786)" + + This reverts commit f6ffa0014213b70cdafcd4d817e6b6265d76d277. + +commit f6ffa0014213b70cdafcd4d817e6b6265d76d277 +Author: István Zoltán Szabó +Date: Thu Aug 22 15:04:20 2019 +0200 + + [DOCS] Reformats search API (#45786) + + Co-Authored-By: James Rodewig + +commit 5e44e695fdca63d32b9753323b0fb0b65d3ca778 +Author: James Rodewig +Date: Thu Aug 22 08:40:09 2019 -0400 + + [DOCS] Add template docs to scripts. Reorder template examples. (#45817) + + * [DOCS] Add template docs to scripts. Reorder template examples. + + * Adds a 'Search template' section to the 'How to use scripts' chapter. + This links to the 'Search template' chapter for detailed info and + examples. + + * Reorders and retitles several examples in the 'Search template' + chapter. This is primarily to make examples for storing, deleting, and + using search templates more prominent. + + * Change to + +commit ed8307c198db6e1f775ee39ccb5c9baf729a0b31 +Author: Andrey Ershov +Date: Thu Aug 22 13:04:38 2019 +0200 + + Deprecate es.http.cname_in_publish_address setting (#45616) + + Follow up on #32806. + + The system property es.http.cname_in_publish_address is deprecated + starting from 7.0.0 and deprecation warning should be added if the + property is specified. + This PR will go to 7.x and master. + Follow-up PR to remove es.http.cname_in_publish_address property + completely will go to the master. + + (cherry picked from commit a5ceca7715818f47ec87dd5f17f8812c584b592b) + +commit 57a36eb37351a5c3df79f83ed711d85d742dd529 +Author: Tanguy Leroux +Date: Thu Aug 22 11:27:54 2019 +0200 + + Add tests to check that requests are retried when writing/reading blobs on S3 (#45383) + + This commit adds tests to verify the behavior of the S3BlobContainer and + its underlying AWS SDK client when the remote S3 service is responding + errors or not responding at all. The expected behavior is that requests are + retried multiple times before the client gives up and the S3BlobContainer + bubbles up an exception. + + The test verifies the behavior of BlobContainer.writeBlob() and + BlobContainer.readBlob(). In the case of S3 writing a blob can be executed + as a single upload or using multipart requests; the test checks both scenario + by writing a small then a large blob. + +commit 88acae48ced807578b479d09a78c8cbf85a9e35b +Author: Armin Braun +Date: Thu Aug 22 11:32:45 2019 +0200 + + Remove index-N Rebuild in Shard Snapshot Updates (#45740) (#45778) + + * There is no point in listing out every shard over and over when the `index-N` blob in the shard contains a list of all the files + * Rebuilding the `index-N` from the `snap-${uuid}.dat` blobs does not provide any material benefit. It only would in the corner case of a corrupted `index-N` but otherwise uncorrupted blobs since we neither check the correctness of the content of all segment blobs nor do we do a similar recovery at the root of the repository. + * Also, at least in version `6.x` we only mark a shard snapshot as successful after writing out the updated `index-N` blob so all snapshots that would work with `7.x` and newer must have correct `index-N` blobs + + => Removed the rebuilding of the `index-N` content from `snap-${uuid}.dat` files and moved to only listing `index-N` when taking a snapshot instead of listing all files + => Removed check of file existence against physical blob listing + => Kept full listing on the delete side to retain full cleanup of blobs that aren't referenced by the `index-N` + +commit 75123379226f89c84dc0f2ccb47707a4f01a080b +Author: Przemysław Witek +Date: Thu Aug 22 11:14:26 2019 +0200 + + [7.x] Allow the user to specify 'query' in Evaluate Data Frame request (#45775) (#45825) + +commit b95ca9c3bb2fc7c75b999da71d7d3a69297980f0 +Author: Luca Cavanna +Date: Thu Aug 22 11:13:26 2019 +0200 + + Fix compile errors in HttpChannelTaskHandler + + Relates to #43332 + +commit a47ade3e64a98adc9bf72affd49c3e48c614be93 +Author: Luca Cavanna +Date: Wed Aug 21 19:01:37 2019 +0200 + + Cancel search task on connection close (#43332) + + This PR introduces a mechanism to cancel a search task when its corresponding connection gets closed. That would relief users from having to manually deal with tasks and cancel them if needed. Especially the process of finding the task_id requires calling get tasks which needs to call every node in the cluster. + + The implementation is based on associating each http channel with its currently running search task, and cancelling the task when the previously registered close listener gets called. + +commit 33972423e9b586975c66942e236f03d3955a8b3c +Author: Martijn van Groningen +Date: Thu Aug 22 09:22:40 2019 +0200 + + Enrich processor configuration changes (#45466) + + Enrich processor configuration changes: + * Renamed `enrich_key` option to `field` option. + * Replaced `set_from` and `targets` options with `target_field`. + + The `target_field` option behaves different to how `set_from` and + `targets` worked. The `target_field` is the field that will contain + the looked up document. + + Relates to #32789 + +commit 824f1090a92d73edb323bef42205202690c34e93 +Author: Armin Braun +Date: Thu Aug 22 06:06:09 2019 +0200 + + Disable testTimeoutPerConnection on Windows (#45785) (#45818) + + * It appears this test that is specific to how the BSD network stack works + does randomly fail on Windows => disabling it since it's not clear that it + should work on Windows in a stable way + * Fixes #45777 + +commit 302988745110baa401885556289c0192595404b4 +Author: Nhat Nguyen +Date: Wed Aug 21 18:16:58 2019 -0400 + + Never release store using CancellableThreads (#45409) + + Today we can release a Store using CancellableThreads. If we are holding + the last reference, then we will verify the node lock before deleting + the store. Checking node lock performs some I/O on FileChannel. If the + current thread is interrupted, then the channel will be closed and the + node lock will also be invalid. + + Closes #45237 + +commit 9b180314e3cadbe7d398b714547192af5fdf4fcb +Author: debadair +Date: Tue Aug 20 09:14:49 2019 -0700 + + [DOCS] Streamlined GS indexing topic. (#45714) + + * Streamlined GS indexing topic. + + * Incorporated review feedback + + * Applied formatting per the style guidelines. + +commit cff09bea00ea32592f14bc016b447c76e7d6d298 +Author: Ryan Ernst +Date: Wed Aug 21 14:18:26 2019 -0700 + + Improve distro tests base class (#45761) (#45803) + + This commit moves many features of individual distro tests into the base + class so that other test cases can utilize them. It also standardizes + the pattern for tests adding assumptions for the particular + distributions to test. + +commit 9b14b7298b4a3add4eee4a699bcee22d8d56eecb +Author: Tal Levy +Date: Wed Aug 21 14:15:49 2019 -0700 + + [7.x] Add is_write_index column to cat.aliases (#45798) + + * Add is_write_index column to cat.aliases (#44772) + + Aliases have had the option to set `is_write_index` since 6.4, + but the cat.aliases action was never updated. + + * correct version bounds to 7.4 + +commit 3ebeaa255730e7e78c26b9349741d44f68f0f555 +Author: Benjamin Trent +Date: Wed Aug 21 14:46:09 2019 -0500 + + Fixing rollup state tests after onFailure ordering change (#45784) (#45814) + + After the PR #45676 onFailure is now called before the indexer state has transitioned out of indexing. + + To fix these tests, I added a new check to make sure that we don't mark it as failed until AFTER doSaveState is called with a STARTED indexer. + +commit 47b1e2b3d06eb7fedc6327edccc04b2548e2228d +Author: Gordon Brown +Date: Wed Aug 21 13:42:11 2019 -0600 + + [7.x] Use rollover for SLM's history indices (#45686) + + Following our own guidelines, SLM should use rollover instead of purely + time-based indices to keep shard counts low. This commit implements lazy + index creation for SLM's history indices, indexing via an alias, and + rollover in the built-in ILM policy. + +commit c3296d3251770f7487db415dfcaa93ed0ac18c78 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Aug 21 20:53:14 2019 +0200 + + Unmute testBiDirectionalIndexFollowing (#45641) (#45792) + + Cause is believed to be in build system caching so unmuting. + +commit 2b549e7342ebb59ca4ad35dd5109f610ad411350 +Author: William Brafford +Date: Wed Aug 21 14:46:07 2019 -0400 + + CLI tools: write errors to stderr instead of stdout (#45586) + + Most of our CLI tools use the Terminal class, which previously did not provide methods for writing to standard output. When all output goes to standard out, there are two basic problems. First, errors and warnings are "swallowed" in pipelines, making it hard for a user to know when something's gone wrong. Second, errors and warnings are intermingled with legitimate output, making it difficult to pass the results of interactive scripts to other tools. + + This commit adds a second set of print commands to Terminal for printing to standard error, with errorPrint corresponding to print and errorPrintln corresponding to println. This leaves it to developers to decide which output should go where. It also adjusts existing commands to send errors and warnings to stderr. + + Usage is printed to standard output when it's correctly requested (e.g., bin/elasticsearch-keystore --help) but goes to standard error when a command is invoked incorrectly (e.g. bin/elasticsearch-keystore list-with-a-typo | sort). + +commit 2b8d79c470f8060a68902546aff2e51e60ff5171 +Author: Ryan Ernst +Date: Wed Aug 21 11:25:10 2019 -0700 + + Add destructiveDistroTest meta task (#45762) + + This commit adds a destructiveDistroTest task which depends on all of + the distribution specific destructive tasks, which can be used by CI. + + closes #45769 + +commit 790765d3f985db03b43907a55612560266a7f013 +Author: Armin Braun +Date: Wed Aug 21 19:26:19 2019 +0200 + + Remove Dep. on SnapshotsService in SnapshotShardsService (#45776) (#45791) + + SnapshotShardsService depends on the RepositoriesService + not the SnapshotsService, no need to have this indirection. + +commit 6aaee8aa0ae481611e25e9ff0b7fcd90ca750970 +Author: Armin Braun +Date: Wed Aug 21 17:59:49 2019 +0200 + + Repository Cleanup Endpoint (#43900) (#45780) + + * Repository Cleanup Endpoint (#43900) + + * Snapshot cleanup functionality via transport/REST endpoint. + * Added all the infrastructure for this with the HLRC and node client + * Made use of it in tests and resolved relevant TODO + * Added new `Custom` CS element that tracks the cleanup logic. + Kept it similar to the delete and in progress classes and gave it + some (for now) redundant way of handling multiple cleanups but only allow one + * Use the exact same mechanism used by deletes to have the combination + of CS entry and increment in repository state ID provide some + concurrency safety (the initial approach of just an entry in the CS + was not enough, we must increment the repository state ID to be safe + against concurrent modifications, otherwise we run the risk of "cleaning up" + blobs that just got created without noticing) + * Isolated the logic to the transport action class as much as I could. + It's not ideal, but we don't need to keep any state and do the same + for other repository operations + (like getting the detailed snapshot shard status) + +commit fe2a7523ec0b7154973a8ff08e0ea6d78db198b1 +Author: Jim Ferenczi +Date: Tue Aug 20 15:06:01 2019 +0200 + + Add support for inlined user dictionary in the Kuromoji plugin (#45489) + + This change adds a new option called user_dictionary_rules to + Kuromoji's tokenizer. It can be used to set additional tokenization rules + to the Japanese tokenizer directly in the settings (instead of using a file). + This commit also adds a check that no rules are duplicated since this is not allowed + in the UserDictionary. + + Closes #25343 + +commit 3318c91fea31557ccade4d515d8d2324a0345582 +Author: James Rodewig +Date: Wed Aug 21 09:42:10 2019 -0400 + + [DOCS] Reformat get mapping API. Reformat and reuse multi-index params. (#45699) + +commit 7f2ba913606599933c08b02e12c810a5a421adf6 +Author: Martijn van Groningen +Date: Fri Aug 16 09:58:14 2019 +0200 + + adjusted enrich rest specs to new format + +commit 2677ac14d210bf8b1b35795966e3d5085aebd3de +Merge: f28934f7459 bf701b83d21 +Author: Martijn van Groningen +Date: Wed Aug 21 14:28:17 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 5faa012fd6c1cd7b04fa6bfa77173ca5f87f29f5 +Author: Przemysław Witek +Date: Wed Aug 21 14:27:36 2019 +0200 + + [7.x] Add docs for HLRC for Estimate memory usage API (#45538) (#45783) + +commit bf701b83d218226ab347c5543b536d7cb5339200 +Author: Przemysław Witek +Date: Wed Aug 21 12:45:09 2019 +0200 + + Shorten field names in EstimateMemoryUsageResponse (#45719) (#45772) + +commit 6b391cd0d59e7fd828f7550893fb36820437c57d +Author: Zachary Tong +Date: Thu Aug 15 13:37:15 2019 -0400 + + Mute ShapeQueryTests#testFieldAlias() + + Tracking issue: https://github.com/elastic/elasticsearch/issues/45628 + +commit f28934f7459d236a4900e11983956bf451866b33 +Author: Martijn van Groningen +Date: Wed Aug 21 11:18:50 2019 +0200 + + fixed test compile error + +commit 982560afebc78abe1af2f1804e8319d6ffc0332e +Author: David Kyle +Date: Wed Aug 21 10:02:45 2019 +0100 + + Mute RollupIndexerStateTests + + See #45770 + +commit 1d9b2d57ce1e6e77951e412dee97ac6ee8f3aff2 +Author: Russ Cam +Date: Wed Aug 21 18:47:50 2019 +1000 + + Use the elasticsearch.bat file in zip distribution (#45596) + + This commit updates the documentation to + + - use the batch file included with the zip distribution; the exe file is included in the MSI only. + - introduce a space between the -E arguments and their values. Without a space (or quoted, but adding a space is cleaner), the argument will fail with PowerShell + + (cherry picked from commit 5c8dbcedb0edf3a48ca1ec52aad9ea41fa941f8a) + +commit 5864f30771684871ded0e9158de4f700ff3b6453 +Author: Martijn van Groningen +Date: Wed Aug 21 10:06:26 2019 +0200 + + ensure that the items in the bulk response are the same as is in the bulk request + +commit c6709f0979e95e0cccdae012894c4dc7e6e5ae8b +Author: Przemysław Witek +Date: Wed Aug 21 09:57:23 2019 +0200 + + Mute tests affected by renaming fields in Estimate memory usage response (#45743) (#45766) + +commit d5c3d9b50f9918f89ef4b4dd86af00f2880b9d26 +Author: Dimitris Athanasiou +Date: Wed Aug 21 08:15:38 2019 +0300 + + [7.x][ML] Do not skip rows with missing values for regression (#45751) (#45754) + + Regression analysis support missing fields. Even more, it is expected + that the dependent variable has missing fields to the part of the + data frame that is not for training. + + This commit allows to declare that an analysis supports missing values. + For such analysis, rows with missing values are not skipped. Instead, + they are written as normal with empty strings used for the missing values. + + This also contains a fix to the integration test. + + Closes #45425 + +commit ba7b6776181cf656b72a83c8dd578f4ec9dc5834 +Author: Benjamin Trent +Date: Tue Aug 20 17:37:04 2019 -0500 + + [ML] better handle empty results when evaluating regression (#45745) (#45759) + + * [ML] better handle empty results when evaluating regression + + * adding new failure test to ml_security black list + + * fixing equality check for regression results + +commit 686739d45638465021c94686e207ae91c954d961 +Author: Denis Derezhenets <1963048+9deniz@users.noreply.github.com> +Date: Tue Aug 20 22:58:58 2019 +0200 + + Added slices parameter to HLRC reindex request + +commit 18fb63209bf8beb11323c3bda932212b7878e667 +Author: Ryan Ernst +Date: Tue Aug 20 13:12:15 2019 -0700 + + Separate distro tests to be per distribution (#45565) + + The java based distribution tests currently have a single Tests class + which encapsulates all of the tests for a particular distribution. The + test task in gradle then depends on all distributions being built, and + each individual tests class looks for the particular distribution it is + trying to test. This means that reproducing a single test failure + triggers all the distributions to be built, even though only one is + needed for the test. + + This commit reworks the java distribution tests to pass in a particular + distribution to be tested, and changes the base test classes to be + actual test classes which have assumptions around which distributions + they operate on. For example, the archives tests will be skipped when + run with an rpm distribution, and vice versa for the package tests. This + makes reproduction much more granular. It also also better splitting up + tests around a particular use case. For example, all tests for systemd + behavior can be in one test class, and run independently of all tests + against rpm/deb distributions. + +commit 2a0c7c40e52f8608165e4d3ac8d3863046d2e6c5 +Author: Christos Soulios +Date: Tue Aug 20 20:19:51 2019 +0300 + + [7.x] Implement AvgAggregatorTests#testDontCacheScripts and remove AvgIT #45746 + + Backports PR #45737: + + Similar to PR #45030 integration test testDontCacheScripts() was moved to unit test AvgAggregatorTests#testDontCacheScripts. + + AvgIT class was removed. + +commit a01bd6c5a38a3030202344a263069f310311b8e5 +Author: Armin Braun +Date: Tue Aug 20 19:15:36 2019 +0200 + + Stop Executing SLM Policy Transport Action on Snapshot Pool (#45727) (#45748) + + * Executing SLM policies on the snapshot thread will block until a snapshot finishes if the pool is completely busy executing that snapshot + * Fixes #45594 + +commit a27759a624aa0cce0fc6aa13d2427ae5c0833a23 +Author: James Rodewig +Date: Tue Aug 20 12:58:57 2019 -0400 + + [DOCS] Document `indices.lifecycle.poll_interval` ILM cluster setting (#45744) + +commit ac7173c0d48e709e10d1fd6a385414fa83c35ed7 +Author: Martijn van Groningen +Date: Tue Aug 20 18:51:01 2019 +0200 + + Renamed CoordinatorProxyAction to EnrichCoordinatorProxyAction and (#45663) + + fail if query shard context needs current time (certain queries / scripts + use this, but in the enrich context this is not used). + +commit 96a40acd822756a1ba22e5e1420331a2963b7c80 +Author: Christos Soulios +Date: Tue Aug 20 18:58:47 2019 +0300 + + [7.x] Migrate tests from MaxIT to MaxAggregatorTests (#45030) #45742 + + Backports PR #45030 to 7.x: + + This PR migrates tests from MaxIT integration test to MaxAggregatorTests, as described in #42893 + +commit e3373d349b18dd54cf7016183918bee9d4c7f51b +Author: Michael Basnight +Date: Tue Aug 20 10:05:45 2019 -0500 + + Consolidate enrich list all and get by name APIs (#45705) + + The get and list APIs are a single API in this commit. Whether + requesting one named policy or all policies, a list of policies is + returened. The list API code has all been removed and the GET api is + what remains, which contains much of the list response code. + +commit a219a0f819675325001d021ac33692b03a379af2 +Author: James Rodewig +Date: Tue Aug 20 10:36:42 2019 -0400 + + [DOCS] Replace placeholder anchors for create rollup job API docs. (#45712) + +commit e9759b2b331fbeda25b3a54748cf616bd0421afb +Author: Nhat Nguyen +Date: Tue Aug 20 09:13:46 2019 -0400 + + Wait for background refresh in testAutomaticRefresh (#45661) + + If the background refresh is running, but not finished yet then the + document might not be visible to the next search. Thus, if + scheduledRefresh returns false, we need to wait until the background + refresh is done. + + Closes #45571 + +commit 99b21d50b849f146eb7bd4e0d90d2f67a87ec11c +Author: Nhat Nguyen +Date: Mon Aug 19 09:43:28 2019 -0400 + + Include leases in ccr errmsg when ops no longer available (#45681) + + The setting index.soft_deletes.retention.operations is no longer needed + nor recommended in CCR. We, therefore, should hint users about the + retention leases period setting instead when operations are no longer + available for replicating. + +commit 43bb5924e68dc013013e842a9efd5cd29de4845d +Author: Benjamin Trent +Date: Tue Aug 20 09:23:07 2019 -0500 + + [ML][Data Frame] fixing _start?force=true bug (#45660) (#45734) + + * [ML][Data Frame] fixing _start?force=true bug + + * removing unused import + + * removing old TODO + +commit 49edf9e5b5fcc9d0a2f2ecf315e2945e49e26624 +Author: Dimitris Athanasiou +Date: Tue Aug 20 17:21:40 2019 +0300 + + [7.x][ML] Remove timeout on waiting for DF analytics result processor to complete (#45724) (#45733) + + We cannot know how long the analysis will take to complete thus we should not have + a timeout. Note that if the process crashes, the result processor will pick the + exception due to the stream closing. + + Closes #45723 + +commit b37ebd1adf4ffad1a4733d6da740d8c797a744c3 +Author: Przemysław Witek +Date: Tue Aug 20 16:03:50 2019 +0200 + + Prepare the codebase for new Auditor subclasses (#45716) (#45731) + +commit 80dd0a0948e7158da1114587b8ea15470b5373bd +Author: Przemysław Witek +Date: Tue Aug 20 15:49:17 2019 +0200 + + Get rid of EstimateMemoryUsageRequest and EstimateMemoryUsageAction.Request. (#45718) (#45725) + +commit 88641a08afb9afa00fe32da94a87e9ff9871200b +Author: Benjamin Trent +Date: Tue Aug 20 07:30:17 2019 -0500 + + [ML][Data frame] fixing failure state transitions and race condition (#45627) (#45656) + + * [ML][Data frame] fixing failure state transitions and race condition (#45627) + + There is a small window for a race condition while we are flagging a task as failed. + + Here are the steps where the race condition occurs: + 1. A failure occurs + 2. Before `AsyncTwoPhaseIndexer` calls the `onFailure` handler it does the following: + a. `finishAndSetState()` which sets the IndexerState to STARTED + b. `doSaveState(...)` which attempts to save the current state of the indexer + 3. Another trigger is fired BEFORE `onFailure` can fire, but AFTER `finishAndSetState()` occurs. + + The trick here is that we will eventually set the indexer to failed, but possibly not before another trigger had the opportunity to fire. This could obviously cause some weird state interactions. To combat this, I have put in some predicates to verify the state before taking actions. This is so if state is indeed marked failed, the "second trigger" stops ASAP. + + Additionally, I move the task state checks INTO the `start` and `stop` methods, which will now require a `force` parameter. `start`, `stop`, `trigger` and `markAsFailed` are all `synchronized`. This should gives us some guarantees that one will not switch states out from underneath another. + + I also flag the task as `failed` BEFORE we successfully write it to cluster state, this is to allow us to make the task fail more quickly. But, this does add the behavior where the task is "failed" but the cluster state does not indicate as much. Adding the checks in `start` and `stop` will handle this "real state vs cluster state" race condition. This has always been a problem for `_stop` as it is not a master node action and doesn’t always have the latest cluster state. + + closes #45609 + + Relates to #45562 + + * [ML][Data Frame] moves failure state transition for MT safety (#45676) + + * [ML][Data Frame] moves failure state transition for MT safety + + * removing unused imports + +commit 041385559c97db3862822ac789774e53568ecb9d +Author: Jonathan Hult +Date: Tue Aug 20 07:27:27 2019 -0400 + + [DOCS] Fix typo in highlighting doc (#45707) + +commit 7d5ab17bb285924f282464879ed9700692595191 +Author: markharwood +Date: Tue Aug 20 11:38:22 2019 +0100 + + Search enhancement: pinned queries (#44345) (#45657) + + * Search enhancement: pinned queries (#44345) + + Search enhancement: - new query type allows selected documents to be promoted above any "organic” search results. + This is the first feature in a new module `search-business-rules` which will house licensed (non OSS) logic for rewriting queries according to business rules. + The PinnedQueryBuilder class offers a new `pinned` query in the DSL that takes an array of promoted IDs and an “organic” query and ensures the documents with the promoted IDs rank higher than the organic matches. + + Closes #44074 + +commit 0f51dd69cb82f327d32eb176bcf8aeebf625de30 +Author: Costin Leau +Date: Tue Aug 20 09:33:18 2019 +0300 + + SQL: Improve serialization of SQL processors (#45678) + + Encapsulate the serialization/deserialization of SQL client classes. + Make configuration specific parameters (such as ZoneId) generic just + like the version and remove the need for consumer classes to manage them + individually. + This is not only consistent but also provides significant savings in the + cursor. + + Fix #40216 + + (cherry picked from commit 5c844798045d7baa0d932289d2e3d1607ba6a9a4) + +commit c6b30b8883b4767af9a5103e68a8176336359809 +Author: Alpar Torok +Date: Tue Aug 20 10:01:03 2019 +0300 + + Add input and outut tracking of built bwc versions (#45694) + + * Add input and outut tracking of built bwc versions + + This PR adds tracking of the bwc versions git has as input and all the + expected files as output. + The effect is that `gradlew` is not called at all when the git has + doesn't change and the version was allready built. + Previusly gradlew would be called for the bwc version and it would have + to configure the project and go trough up to date checks to figure out + that nothing changed. + This helps when working on bwc tests locally needing to run the test + multiple times. + This should also help in CI not re-build bwc versions across different + runs. + + * Enable caching of bwc builds + +commit 1818c5fa448b8b6df792b495a6452fe0065a0e79 +Author: Igor Motov +Date: Mon Aug 19 18:16:32 2019 -0400 + + Ingest Attachment: Upgrade tika to v1.22 (#45575) + + Upgrades: + Apache Tika: 1.19.1 -> 1.22. + pdfbox : 2.0.12 -> 2.0.16 + poi : 4.0.0 -> 4.0.1 + +commit 5dcc00a8b33ddfc5d22a4027e29ea3069126d4e0 +Author: James Rodewig +Date: Mon Aug 19 16:54:39 2019 -0400 + + [DOCS] Add placeholder for 7.3.1 release notes (#45710) + +commit 7bc8400222ece398be5861e2e98691b206b4c172 +Author: Przemysław Witek +Date: Mon Aug 19 21:37:55 2019 +0200 + + Call the new _estimate_memory_usage API endpoint on df analytics _start (#45536) (#45701) + +commit 8f86faca5cf12b84ed3d43d83b7aca2cba5bed9a +Author: LHearen +Date: Mon Aug 19 22:07:37 2019 +0800 + + [DOCS] Correct conditional clause in histogram agg docs (#45643) + +commit da0a785685b5a77e93ed0444fc3d49a5bb753c43 +Author: LHearen +Date: Mon Aug 19 22:02:09 2019 +0800 + + [DOCS] Fix a 'value' -> 'values' typo in histogram aggregation docs (#45642) + +commit b0ef313817743454452b3c7c8da221c9cff3df86 +Author: James Rodewig +Date: Mon Aug 19 09:32:37 2019 -0400 + + [DOCS] Add examples to the mapping docs (#45520) + +commit 4b932519aabaf1c2cfb8abfba884e3b5c91e2a17 +Author: James Rodewig +Date: Mon Aug 19 08:27:03 2019 -0400 + + [DOCS] Document `throttle_period_in_millis` for watcher actions (#45607) + +commit 47b3dccbc40c553b02e84c5b286fa8c7a06593a9 +Author: Rory Hunter +Date: Mon Aug 19 10:29:41 2019 +0100 + + Always check that cgroup data is present (#45647) + + `OsProbe` fetches cgroup data from the filesystem, and has asserts that + check for missing values. This PR changes most of these asserts into + runtime checks, since at least one user has reported an NPE where + a piece of cgroup data was missing. + + Backport of #45606 to 7.x. + +commit 6f5d944fbd3484d3b8f55824617ed299f938d95d +Author: Nhat Nguyen +Date: Sat Aug 17 13:47:09 2019 -0400 + + Ensure AsyncTask#isScheduled remain false after close (#45687) + + If a scheduled task of an AbstractAsyncTask starts after it was closed, + then isScheduledOrRunning can remain true forever although no task is + running or scheduled. + + Closes #45576 + +commit 6f2daa85e3c5da7cef095dd776bb91c9746a0dde +Author: Vega <603585402@qq.com> +Date: Sat Aug 17 08:48:56 2019 +0800 + + Allow uppercase in keystore setting names (#45222) + + The elasticsearch keystore was originally backed by a PKCS#12 keystore, which had several limitations. To overcome some of these limitations in encoding, the setting names existing within the keystore were limited to lowercase alphanumberic (with underscore). Now that the keystore is backed by an encrypted blob, this restriction is no longer relevant. This commit relaxes that restriction by allowing uppercase ascii characters as well. + + closes #43835 + +commit 200579bfce5452c07532b73cee5df664d5a980a4 +Author: Jason Tedor +Date: Fri Aug 16 18:32:12 2019 -0400 + + Simplify finding jps (#45677) + + This commit simplifies how we find jps to instead use built-in Gradle + functionality for the same. + +commit 74de6d2c711e715568d8f962c6915bab504aeca9 +Author: debadair +Date: Fri Aug 16 14:54:21 2019 -0700 + + [DOCS] Fixed TOC heading level issue. (#45680) + +commit 1cd58c8ea8d8f081cda02eee111a60232f5c2eb9 +Author: Costin Leau +Date: Fri Aug 16 17:49:25 2019 +0300 + + SQL: Break TextFormatter/Cursor dependency (#45613) + + Improve the initialization and state passing of TextFormatter in CLI + and TEXT mode by leveraging the Page listener hook. Additionally + simplify the code inside RestSqlQueryAction. + + (cherry picked from commit a56db2fa119cf9e8748723e19f1fc9f6a8afe5fc) + +commit 96883dd028059a4f5849ed2150cd482bad4ab1a8 +Author: Costin Leau +Date: Fri Aug 16 15:39:36 2019 +0300 + + SQL: Refactor away the cycle between Rowset and Cursor (#45516) + + Improve encapsulation of pagination of rowsets by breaking the cycle + between cursor and associated rowset implementation, all logic now + residing inside each cursor implementation. + + (cherry picked from commit be8fe0a0ce562fe732fae12a0b236b5731e4638c) + +commit ecb3ebd7962f1b4814d8b048945446fce5ccad80 +Author: Gordon Brown +Date: Fri Aug 16 14:17:34 2019 -0600 + + Clean SLM and ongoing snapshots in test framework (#45564) + + Adjusts the cluster cleanup routine in ESRestTestCase to clean up SLM + test cases, and optionally wait for all snapshots to be deleted. + + Waiting for all snapshots to be deleted, rather than failing if any are + in progress, is necessary for tests which use SLM policies because SLM + policies may be in the process of executing when the test ends. + +commit c321272ae76360154117d341a8c1cb9231ec9ce5 +Author: Armin Braun +Date: Fri Aug 16 22:02:41 2019 +0200 + + Mute testBiDirectionalIndexFollowing for #45641 (#45674) + + * Muting #45641 + +commit 98c850c08bbed147eee4e8fe3b96e71957c2d4da +Author: Igor Motov +Date: Fri Aug 16 14:42:02 2019 -0400 + + Geo: Change order of parameter in Geometries to lon, lat 7.x (#45618) + + Changes the order of parameters in Geometries from lat, lon to lon, lat + and moves all Geometry classes are moved to the + org.elasticsearch.geomtery package. + + Backport of #45332 + + Closes #45048 + +commit 742213d7105dddf91d4bc414e1ef72b198109541 +Author: Ryan Ernst +Date: Fri Aug 16 11:39:03 2019 -0700 + + Improve error message when index settings are not a map (#45588) + + This commit adds an explicit error message when a create index request + contains a settings key that is not a json object. Prior to this change + the user would be given a ClassCastException with no explanation of what + went wrong. + + closes #45126 + +commit 50c65d05baa5e62d3d600b2bea4d76e5d1acc661 +Author: Zachary Tong +Date: Fri Aug 16 13:38:52 2019 -0400 + + Move bucket reduction from Bucket to the InternalAgg (#45566) + + The current idiom is to have the InternalAggregator find all the + buckets sharing the same key, put them in a list, get the first bucket + and ask that bucket to reduce all the buckets (including itself). + + This a somewhat confusing workflow, and feels like the aggregator should + be reducing the buckets (since the aggregator owns the buckets), rather + than asking one bucket to do all the reductions. + + This commit basically moves the `Bucket.reduce()` method to the + InternalAgg and renames it `reduceBucket()`. It also moves the + `createBucket()` (or equivalent) method from the bucket to the + InternalAgg as well. + +commit 2ecd6a83f72c1d1c15d0e8392b18c017b79c779c +Author: Jason Tedor +Date: Fri Aug 16 13:38:26 2019 -0400 + + Clarify mixed license text (#45637) + + This commit clarifies the mixed licensing in the elasticsearch + repository. A few points of note: + - we clarify that all code is under the Apache License 2.0 unless noted + otherwise + - we clarify that code under the Elastic License is only in the x-pack + directory + - we clarify that when code is not under the Apache License 2.0 nor the + Elastic License that it is Apache License 2.0 compatible + - we clarify that OSS-builds do not package any code from the x-pack + directory + +commit 0e375e6cd77447e108a2322bb99e369747ada45b +Author: Mark Vieira +Date: Fri Aug 16 10:30:14 2019 -0700 + + Disable test runner task when not running with testclusters (#45659) + +commit e243bbdc2a6566dce2c3b9037d02588a26beaf44 +Author: Jack Conradson +Date: Fri Aug 16 08:32:18 2019 -0700 + + Fix Watcher Examples in Painless (#45631) + + This fixes the mappings and types required to run watcher and other + examples. A new set of seat data will be updated and available for + download to go with this change. + +commit 529946aa15d8e906c07062cd60dccb0ed3817073 +Author: Mark Vieira +Date: Fri Aug 16 08:43:14 2019 -0700 + + Rename system property to change bwc checkout behavior (#45574) + +commit dbc90653dcba49db13eaa1ff8a7727685648713b +Author: Andrey Ershov +Date: Fri Aug 16 18:40:04 2019 +0200 + + transport.publish_address should contain CNAME (#45626) + + This commit adds CNAME reporting for transport.publish_address same way + it's done for http.publish_address. + + Relates #32806 + Relates #39970 + + (cherry picked from commit e0a2558a4c3a6b6fbfc6cd17ed34a6f6ef7b15a9) + +commit 59d378fc7b81bea2a5965f46c61d13fe9d826aef +Author: Alpar Torok +Date: Fri Aug 16 16:43:50 2019 +0300 + + Toggle flag to tail cluster log on failure + + The flag was toggled by accident in a refactoring. + This PR configures it so that we tail the cluster log on failure again. + +commit cab2eeeafe849ac58ce4e0442170e39df93fbead +Author: István Zoltán Szabó +Date: Fri Aug 16 15:41:15 2019 +0200 + + [DOCS] Reformats voting configuration exclusion API. (#45644) + +commit 6350020401b75dc2b8a7a4be0a3821153bdcc7b3 +Author: István Zoltán Szabó +Date: Fri Aug 16 15:38:09 2019 +0200 + + [DOCS] Reformats cluster allocation explain API (#45603) + + Co-Authored-By: James Rodewig + +commit 95bd26444930de81be8b30a4a6bbbf5fff4bd450 +Author: István Zoltán Szabó +Date: Fri Aug 16 15:34:19 2019 +0200 + + [DOCS] Reformats nodes hot_threads API (#45597) + + Co-Authored-By: James Rodewig + +commit c31cddf27e9618582b3cd175e4e0bce320af6a14 +Author: Luca Cavanna +Date: Fri Aug 16 14:40:00 2019 +0200 + + Update the schema for the REST API specification (#42346) + + * Update the REST API specification + + This patch updates the REST API spefication in JSON files to better encode deprecated entities, + to improve specification of URL paths, and to open up the schema for future extensions. + + Notably, it changes the `paths` from a list of strings to a list of objects, where each + particular object encodes all the information for this particular path: the `parts` and the `methods`. + + Among the benefits of this approach is eg. encoding the difference between using the `PUT` and `POST` + methods in the Index API, to either use a specific document ID, or let Elasticsearch generate one. + + Also `documentation` becomes an object that supports an `url` and also a `description` which is a + new field. + + * Adapt YAML runner to new REST API specification format + + The logic for choosing the path to use when running tests has been + simplified, as a consequence of the path parts being listed under each + path in the spec. The special case for create and index has been removed. + + Also the parsing code has been hardened so that errors are thrown earlier + when the structure of the spec differs from what expected, and their + error messages should be more helpful. + +commit 30a07117771c3e33ac5e924fe05d3c335427fd97 +Author: Andrei Stefan +Date: Fri Aug 16 14:58:01 2019 +0300 + + Remove deprecated use of "interval" method, in favor of "fixedInterval". (#45501) + + (cherry picked from commit 3fef65160f9e61883e9f8f7f345b814f945e2f4b) + +commit d6a9edea16205b2c5b6b9a12d4be46088131fb62 +Author: Armin Braun +Date: Fri Aug 16 12:27:54 2019 +0200 + + Lower Limit for Maximum Message Size in TcpTransport (#44496) (#45635) + + * Since we're buffering network reads to the heap and then deserializing them it makes no sense to buffer a message that is 90% of the heap size since we couldn't deserialize it anyway + * I think `30%` is a more reasonable guess here given that we can reasonably assume that the deserialized message will be larger than the serialized message itself and processing it will take additional heap as well + +commit da03a5d8d15c70ba24aa0e199d15ef4c8bd90f45 +Author: István Zoltán Szabó +Date: Fri Aug 16 10:21:25 2019 +0200 + + [DOCS] Reformats task management API (#45549) + + This PR updates the task management API to align with the new API reference template. + +commit 5ea098571138e41180212c1727076026e2e9631a +Merge: db57d2206a3 4a67645e5dc +Author: Martijn van Groningen +Date: Fri Aug 16 09:47:11 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 4a67645e5dc16393b147bb1a761862d06cd6ed62 +Author: Alpar Torok +Date: Fri Aug 16 09:15:04 2019 +0300 + + Use dynamic ports for ESSingleNodeTestCase too + + Extends #45601 to cover all tests. + +commit 1f6cd5c6e11cb05ed4c2c99fe352c657d7b13fdb +Author: Mark Vieira +Date: Thu Aug 15 18:39:11 2019 -0700 + + Fix backwards compatibility with pre-Java11 build-tools users (#45633) + +commit a635eca5f83a7bbddc87f3640ad827294aabeeb0 +Author: James Rodewig +Date: Thu Aug 15 16:11:04 2019 -0400 + + Retitle and relocate cross-cluster search docs (#45608) + +commit ec4182590f083276f601b1ad80a13b21ac255792 +Author: Jason Tedor +Date: Thu Aug 15 16:14:57 2019 -0400 + + Use bundled JDK in Sys V init (#45593) + + This commit addresses an issue when trying to using Elasticsearch on + systems with Sys V init and the bundled JDK was not being used. Instead, + we were still inadvertently trying to fallback on the path. This commit + removes that fallback as that is against our intentions for 7.x where we + only support the bundled JDK or an explicit JDK via JAVA_HOME. + +commit 6b72d42cb8653ca603461f6074b8f0d84b179cb7 +Author: Jason Tedor +Date: Thu Aug 15 15:26:18 2019 -0400 + + Read git revision directly from repository (#45547) + + Today we shell out to git rev-parse to read the git revision. Forking + another process is slower than reading the revision directly. This + commit changes to directly read the git revision from the repository, + avoiding to fork another process. + +commit a48242c371c381172b35cdc45d14e3aeb8d5af69 +Author: Armin Braun +Date: Thu Aug 15 21:16:56 2019 +0200 + + Cleanup Redundant TransportLogger Instantiation (#43265) (#45629) + + * This class' methods are all effectively `static` => make them `static` and stop instantiating it needlessly + +commit cd441f690648b3eb9aeaef2cdbbe4b64f5a647c6 +Author: Zachary Tong +Date: Thu Aug 15 14:58:35 2019 -0400 + + Catch AllocatedTask registration failures (#45300) + + When a persistent task attempts to register an allocated task locally, + this creates the Task object and starts tracking it locally. If there + is a failure while initializing the task, this is handled by a catch + and subsequent error handling (canceling, unregistering, etc). + + But if the task fails to be created because an exception is thrown + in the tasks ctor, this is uncaught and fails the cluster update + thread. The ramification is that a persistent task remains in the + cluster state, but is unable to create the allocated task, and the + exception prevents other tasks "after" the poisoned task from starting + too. + + Because the allocated task is never created, the cancellation tools + are not able to remove the persistent task and it is stuck as a + zombie in the CS. + + This commit adds exception handling around the task creation, + and attempts to notify the master if there is a failure (so the + persistent task can be removed). Even if this notification fails, + the exception handling means the rest of the uninitialized tasks + can proceed as normal. + +commit 73e266b2fd75a900bd3ccc8847ff697c3d5c5fc7 +Author: Armin Braun +Date: Thu Aug 15 19:45:17 2019 +0200 + + Fix Failures when Closing Indices in EsBlobStoreRepositoryIntegTestCase (#45532) (#45614) + + * Same issue as in #44754 as far as I can see: in case of async translog persistence we randomly fail to close + * Closes #45335 + * Closes #45334 + +commit de58353722d0cc767cc1149948c2ae9e7e816ace +Author: Armin Braun +Date: Thu Aug 15 19:41:45 2019 +0200 + + Lower Painless Static Memory Footprint (#45487) (#45619) + + * Painless generates a ton of duplicate strings and empty `Hashmap` instances wrapped as unmodifiable + * This change brings down the static footprint of Painless on an idle node by 20MB (after running the PMC benchmark against said node) + * Since we were looking into ways of optimizing for smaller node sizes I think this is a worthwhile optimization + +commit d64c31e43d10be098e8ee0b80171ebb8721e9d55 +Author: James Rodewig +Date: Thu Aug 15 13:23:25 2019 -0400 + + [DOCS] Rewrite cross-cluster seach docs (#45583) + +commit db57d2206a37110f906c9ed892af1e3444d61196 +Author: Michael Basnight +Date: Thu Aug 15 09:41:04 2019 -0500 + + Prevent delete policy for active executing policy (#45472) + + This commit adds a lock to the delete policy, in the same way that the + locking is done for policy execution. It also creates a test to exercise + the delete transport action, and modifies an existing test to provide a + common set of functions for saving and deleting policies. + +commit c75fd40f2c8379cfd01cd06e8e0eec151c1bab4e +Author: James Rodewig +Date: Thu Aug 15 10:59:58 2019 -0400 + + [DOCS] Add diagrams to cross-cluster search documentation (#45569) + +commit 7119e54be5446f1bab690ee651bdfcbb15f93f58 +Author: Alpar Torok +Date: Thu Aug 15 17:03:18 2019 +0300 + + Mute data frame tests on 7.x + + Tracking in #45610 #45609 + +commit 03a1645bc6b8e38c74aa2c0e306ab3d9396aed42 +Author: Alpar Torok +Date: Thu Aug 15 16:14:18 2019 +0300 + + Use dynamic port ranges for ExternalTestCluster (#45601) + + Moves methods added in #44213 and uses them to configure the port range + for `ExternalTestCluster` too. + These were still using `9300-9400` ( teh default ) and running into + races. + +commit 03f45dad5735588abac319b77b933045e26f8d87 +Author: Michael Basnight +Date: Thu Aug 15 06:20:16 2019 -0500 + + Fix policy removal bug in delete policy (#45573) + + The delete policy had a subtle bug in that it would still delete the + policy if pipelines were accessing it, after giving the client back an + error. This commit fixes that and ensures it does not happen by adding + verification in the test. + +commit d40f3718f2429ef7f5ded583bfbc3bccd3ec4d68 +Author: David Roberts +Date: Thu Aug 15 10:49:29 2019 +0100 + + [ML] Muting 5 SSLErrorMessageTests tests on Windows (#45602) + + Due to https://github.com/elastic/elasticsearch/issues/45598 + +commit 1beea3588b29cbf60b1b160fa04316e0e2a01ade +Author: Armin Braun +Date: Thu Aug 15 07:07:52 2019 +0200 + + Make BlobStoreRepository Validation Read master.dat (#45546) (#45578) + + * Fixing this for two reasons: + 1. Why not verify that the seed we wrote is actually there when we can + 2. The AWS S3 SDK started to log a bunch of WARN messages about not fully reading the stream now that we started to abuse the read blob as an `exists` check after removing that method from the blob container + +commit ad3d8c59bd1fed96dc8e54865f0368de05a9384d +Author: debadair +Date: Wed Aug 14 10:59:17 2019 -0700 + + Restructure the GS topics & add redirects. (#45527) + + * Restructure the GS topics & add redirects. + + Co-Authored-By: James Rodewig + + * Update docs/reference/getting-started.asciidoc to fix tests + +commit 0f780f43b962b45068d58a76b2f6a88f49cf5a8b +Author: debadair +Date: Mon Aug 12 18:19:47 2019 -0700 + + [DOCS] Streamlined install of 3 node cluster for GS tutorial (#45009) + + * [DOCS] Revise GS intro and remove redundant conceptual content. Closes #43846. + + * [DOCS] Incorporated feedback. + + * [DOCS] Consolidated archive install for all platforms. + + * [DOCS] Added powershell command. + + * [DOCS] Added health check. + +commit fde5dae387566c73dcbbf90653a265abff4c0b8b +Author: Benjamin Trent +Date: Wed Aug 14 17:26:24 2019 -0500 + + [ML][Data Frame] adjusting change detection workflow (#45511) (#45580) + + * [ML][Data Frame] adjusting change detection workflow + + * adjusting for PR comment + + * disallowing null as an argument value + +commit 647a8308c302ef02d357eb3d352d06f41dc02b6b +Author: Nick Knize +Date: Wed Aug 14 16:35:10 2019 -0500 + + [SPATIAL] Backport new ShapeFieldMapper and ShapeQueryBuilder to 7x (#45363) + + * Introduce Spatial Plugin (#44389) + + Introduce a skeleton Spatial plugin that holds new licensed features coming to + Geo/Spatial land! + + * [GEO] Refactor DeprecatedParameters in AbstractGeometryFieldMapper (#44923) + + Refactor DeprecatedParameters specific to legacy geo_shape out of + AbstractGeometryFieldMapper.TypeParser#parse. + + * [SPATIAL] New ShapeFieldMapper for indexing cartesian geometries (#44980) + + Add a new ShapeFieldMapper to the xpack spatial module for + indexing arbitrary cartesian geometries using a new field type called shape. + The indexing approach leverages lucene's new XYShape field type which is + backed by BKD in the same manner as LatLonShape but without the WGS84 + latitude longitude restrictions. The new field mapper builds on and + extends the refactoring effort in AbstractGeometryFieldMapper and accepts + shapes in either GeoJSON or WKT format (both of which support non geospatial + geometries). + + Tests are provided in the ShapeFieldMapperTest class in the same manner + as GeoShapeFieldMapperTests and LegacyGeoShapeFieldMapperTests. + Documentation for how to use the new field type and what parameters are + accepted is included. The QueryBuilder for searching indexed shapes is + provided in a separate commit. + + * [SPATIAL] New ShapeQueryBuilder for querying indexed cartesian geometry (#45108) + + Add a new ShapeQueryBuilder to the xpack spatial module for + querying arbitrary Cartesian geometries indexed using the new shape field + type. + + The query builder extends AbstractGeometryQueryBuilder and leverages the + ShapeQueryProcessor added in the previous field mapper commit. + + Tests are provided in ShapeQueryTests in the same manner as + GeoShapeQueryTests and docs are updated to explain how the query works. + +commit deab736aad7c310a3574e8b91163b661c0cbbeaa +Author: Chris Dean <53786748+wchrisdean@users.noreply.github.com> +Date: Wed Aug 14 15:59:36 2019 -0500 + + [DOCS] - Updating chunk_size values to fix size value notation. Chunksize41591 (#45552) (#45579) + + * changes to chunk_size #41591 + + * update to chunk size to include ` ` + + * Update docs/plugins/repository-azure.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/reference/modules/snapshots.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/plugins/repository-azure.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/plugins/repository-s3.asciidoc + + Co-Authored-By: James Rodewig + + * edits to fix passive voice + +commit 285f011bbb10e51f6461a9d77ee733d74d44c997 +Author: Mark Vieira +Date: Wed Aug 14 13:16:20 2019 -0700 + + Add build operating system as build scan tag (#45558) + +commit caa2a7738ff8b861aa9c4b5497b299f35d523545 +Author: Chris Dean +Date: Wed Aug 14 15:14:10 2019 -0500 + + Revert "[DOCS] - Updating chunk_size values to fix size value notation. Chunksize41591 (#45552)" + + This reverts commit 8fdbcd7395dbe2dc69712bc7d62ac77ec91f83b2. + +commit e0d84e7178e6a09a7a4ab15bcbd8feb9cc393acd +Author: Armin Braun +Date: Wed Aug 14 21:53:07 2019 +0200 + + Clean up Callback Chains and Duplicate in SnapshotResiliencyTests (#45398) (#45563) + + * It's in the title, follow up to #45233 + * Flatten more listeners into `StepListener` + * Remove duplication from repo and index bootstrap and asserting that the steps execute successfully + +commit 8fdbcd7395dbe2dc69712bc7d62ac77ec91f83b2 +Author: Chris Dean <53786748+wchrisdean@users.noreply.github.com> +Date: Wed Aug 14 13:47:07 2019 -0500 + + [DOCS] - Updating chunk_size values to fix size value notation. Chunksize41591 (#45552) + + * changes to chunk_size #41591 + + * update to chunk size to include ` ` + + * Update docs/plugins/repository-azure.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/reference/modules/snapshots.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/plugins/repository-azure.asciidoc + + Co-Authored-By: James Rodewig + + * Update docs/plugins/repository-s3.asciidoc + + Co-Authored-By: James Rodewig + + * edits to fix passive voice + +commit fd57d3cb292423b26fc398ba57a1ff00bd8e2b67 +Author: Michael Basnight +Date: Wed Aug 14 13:57:47 2019 -0500 + + Fix test broken by policy rename + +commit e31212b0aa8ba589314921d1045bc5b656b4e4ea +Author: Mark Vieira +Date: Wed Aug 14 11:51:41 2019 -0700 + + Remove leftover debugging logic (#45556) + +commit 52a094b1774ea0082a45ee58e6e247f8488fa835 +Author: Michael Basnight +Date: Wed Aug 14 13:43:41 2019 -0500 + + Fail delete policy if pipeline exists (#44438) + + If a pipeline that refrences the policy exists, we should not allow the + policy to be deleted. The user will need to remove the processor from + the pipeline before deleting the policy. This commit adds a check to + ensure that the policy cannot be deleted if it is referenced by any + pipeline in the system. + +commit 0c343d844343e12694333a0d9f53c890bb07028a +Author: Benjamin Trent +Date: Wed Aug 14 13:08:27 2019 -0500 + + [7.x] [ML][Transforms] adjusting stats.progress for cont. transforms (#45361) (#45551) + + * [ML][Transforms] adjusting stats.progress for cont. transforms (#45361) + + * [ML][Transforms] adjusting stats.progress for cont. transforms + + * addressing PR comments + + * rename fix + + * Adjusting bwc serialization versions + +commit 4af6d86c019ffae4d02be3ecea92b8d7d03c40f6 +Author: Ryan Ernst +Date: Wed Aug 14 10:30:41 2019 -0700 + + Rename vagrant project to os (#45509) (#45530) + + The system level tests for our distributions have historically be run in + vagrant, and thus the name of the gradle project has been "vagrant". + However, as we move to running these tests in other environments (eg + GCP) the name vagrant no longer makes sense. This commit renames the + project to "os" (short for operating system), since these tests ensure + all of our distributions run correctly on our supported operating + systems. + +commit 6904778a14188a108d08ae1784dce0551f9722d9 +Author: James Rodewig +Date: Wed Aug 14 13:06:23 2019 -0400 + + [DOCS] Rewrite `fuzzy` query docs (#42078) + +commit 79a13909354d5ac28bafa0232d713d60ac4a112f +Author: Jim Ferenczi +Date: Wed Aug 14 09:52:49 2019 +0200 + + Add mapper-extras and the RankFeatureQuery in the hlrc (#43713) + + This change adds the support for the RankFeatureQuery in the HLRC by + providing an extra dependency on mapper-extras-client. It also removes + the dependency on lang-painless in mapper-extras which is not needed + anymore since the move of the vector field into a dedicated module. + + Closes #43634 + +commit 8142ca82f193c851f6697acd889c4be7575f37d3 +Author: István Zoltán Szabó +Date: Wed Aug 14 18:19:05 2019 +0200 + + [DOCS] Reformats nodes feature usage API (#45539) + + Co-Authored-By: James Rodewig + +commit 0db5af5f6547f75391d54d8877d453a3057e51be +Author: István Zoltán Szabó +Date: Wed Aug 14 17:07:24 2019 +0200 + + [DOCS] Reformats cluster remote info API (#45545) + + Co-Authored-By: James Rodewig + +commit 5f6bc6fc2dbe99e5c44cba159bc2183047c8c2bc +Author: Armin Braun +Date: Wed Aug 14 14:49:38 2019 +0200 + + Prevent Leaking Search Tasks on Exceptions in FetchSearchPhase and DfsQueryPhase (#45500) (#45540) + + * If `counter.onResult` throws an exception we might leak a transport task because the failure is not handled as a phase failure (instead it bubbles up in the transport service eventually hitting the `onFailure` callback again and couting down the `counter` twice). + + Co-authored-by: Jim Ferenczi + +commit 2a70d38439dcb29a83283559e6d2b05d3f00ae9b +Author: Przemysław Witek +Date: Wed Aug 14 13:45:17 2019 +0200 + + [7.x] HLRC for ml/data_frame/analytics/_estimate_memory_usage API (#45531) (#45537) + +commit 43b8ab607dc90686ee2279833d0d28cb6177ff66 +Author: Martijn van Groningen +Date: Wed Aug 14 11:44:31 2019 +0200 + + Improve naming of enrich policy fields. (#45494) + + Renamed `enrich_key` to `match_field` and + renamed `enrich_values` to `enrich_fields`. + + Relates #32789 + +commit 3eb0e59a8345ff612d758f7798681e6477fc942d +Author: Ioannis Kakavas +Date: Wed Aug 14 11:30:05 2019 +0300 + + Remove reference to `accept_default_password` (#45533) + + `xpack.security.authc.accept_default_password` has not been + used since 6.0 but we still referenced it in our docs. + +commit df574e5168df6773204f3db16e6896dba3029f7b +Author: Przemysław Witek +Date: Wed Aug 14 08:26:03 2019 +0200 + + [7.x] Implement ml/data_frame/analytics/_estimate_memory_usage API endpoint (#45188) (#45510) + +commit 84bf98e9cd982c48b0e658395fb3cac56f274a31 +Author: Ryan Ernst +Date: Tue Aug 13 22:50:35 2019 -0700 + + Fix bats distro test to build distributions (#45529) + + The dependency on copying distributions was accidentally masked by an + earlier refactoring. This commit fixes the copyDistributions task to be + run before bats tests run. + +commit 1403a389f854174f653b3d5140b512320a300938 +Author: Mark Vieira +Date: Tue Aug 13 19:06:20 2019 -0700 + + Fix remote cache misses for test tasks (#45521) + + (cherry picked from commit 14486c73128feefc19d1c6a4cad4864d03ecaa9d) + +commit 3f5dab99c3bc008b18d92753e17630cb55f5990e +Author: Gordon Brown +Date: Tue Aug 13 18:23:20 2019 -0600 + + Properly set origin for SLM history store client (#45515) + + The origin was not set properly for the SnapshotHistoryStore client, + resulting in errors when SLM was used when security was enabled. + +commit bba70172621e28a3d827e6ea19a4f98bb28e28bf +Author: Mark Vieira +Date: Tue Aug 13 16:38:32 2019 -0700 + + Restore check part 1 and 2 order mistakenly reversed in [#45098] (#45522) + +commit b17bf70e04bb22b098e27fd8432c341159c67684 +Author: Mark Vieira +Date: Tue Aug 13 16:02:44 2019 -0700 + + Fix remote cache misses for checkstyle tasks (#45512) + +commit 4fcf7bbd0768bffd033a7c71f83e2e6855690597 +Author: Nhat Nguyen +Date: Tue Aug 13 13:46:36 2019 -0400 + + Do not hold writeLock while verifying Lucene/translog + + We should not hold Engine#writeLock while executing + assertConsistentHistoryBetweenTranslogAndLuceneIndex + for this check might acquire Engine#readLock. + + Relates #45461 + +commit 24514275c7d232ad7178d6d50954751a1c1dfe31 +Author: Nhat Nguyen +Date: Tue Aug 13 12:44:36 2019 -0400 + + Get max_seq_no after snapshot translog and Lucene (#45461) + + We should capture max_seq_no after snapshotting translog and Lucene; + otherwise, that max_seq_no can be smaller some operation in translog or + Lucene. With this change, we also hold the Engine#writeLock during this + check so that no indexing can happen. + + Closes #45454 + +commit adf8e2002192fc590fdf6e60fce2cac85da8d55a +Author: Andrei Stefan +Date: Tue Aug 13 23:04:30 2019 +0300 + + SQL: adds format parameter to range queries for constant date comparisons (#45503) + + * Add format parameter to the range queries built for CURRENT_* functions + used in comparison conditions + * Use range queries for date fields equality/non-equality as well. + + (cherry picked from commit c1e81e90f937ee5a002524d632bfce74d76962f9) + +commit 80a3aeaef188ce31ce1c86295d9f2585a884f6be +Author: Ryan Ernst +Date: Tue Aug 13 12:45:29 2019 -0700 + + Split off upgrade and plugins bats tests (#45360) (#45508) + + The bats tests currently require many additional artifacts to be built. + In addition to the current distributions, they need all the plugins to + be installed, as well as a randomly chosen bwc distribution. This commit + splits these two cases into their own bats task, so the dependencies do + not slow down other tasks like distroTests which do not need them. + +commit 452557cf2e6dd4ff03754e431055d41be199e3fb +Author: Martijn van Groningen +Date: Tue Aug 13 20:24:42 2019 +0200 + + Validate policy name like an index name. (#45452) + + The policy name is used to generate the enrich index name. + For this reason, a policy name should be validated in the same way + as index names. + + Relates to #32789 + +commit 884f26a1dcb152c77e0fa5b6ba36442add5d7968 +Author: Ryan Ernst +Date: Tue Aug 13 08:19:18 2019 -0700 + + Make distro test plugin apply to the top level project (#45406) (#45481) + + The distro test plugin was originally designed to be applied within each + subproject, per operating system we run in a VM with vagrant. However, + for efficiency, and also ease of having a single task to run in CI when + launching within individual OS VMs, having the "destructive" tasks in a + single place is more convenient. This commit reworks the distro test + plugin to be applied to the qa/vagrant project, which now creates only + the wrapper tasks in each of the subprojects for each vagrant VM. + +commit 7f550f2b293a1d2e950ec16390193142188d2e34 +Author: Jack Conradson +Date: Tue Aug 13 08:00:11 2019 -0700 + + Complete decoupling ANTLR AST from Painless AST (#45366) + + This change removes the Reserved class used to track variables usages + within the ANTLR grammar. That task is now performed by an existing pass + "extractVariables" in the Painless AST. The Painless AST no longer has any + dependencies on the ANTLR AST for state outside of the tree being built. + This will simplify future refactoring and opens the possibility of alternate + grammars. + +commit 90803a5caf74741cf48d96d6bd2d404ec3c7c051 +Author: Armin Braun +Date: Tue Aug 13 15:55:54 2019 +0200 + + Reenable Integ Tests in native-multi-node-tests (#45482) (#45496) + + * Reenable Integ Tests in native-multi-node-tests + + * The tests broken here were likely fixed by #45463 => let's reenable them and see if things run fine again + * Relates #45405, #45455 + +commit 22ab38953179ebc2a4206f58658095dfd75ad7c7 +Author: Mayya Sharipova +Date: Mon Aug 12 21:14:13 2019 -0400 + + Clarify that FLS/DLS disable shard request cache (#45462) + +commit 109358ca3ff5161ec7e20bc39e18fb938304ebb9 +Author: James Rodewig +Date: Tue Aug 13 08:35:08 2019 -0400 + + [DOCS] Reformat cat API titles (#45407) + +commit df0df9402d42f98fc67eb4fb8aa3c96dd9b71fa3 +Author: James Rodewig +Date: Tue Aug 13 08:27:57 2019 -0400 + + [DOCS] Correct anchor for cat segments API examples + +commit 2543e9eca5b3b6b22b56bf7103acc0d932f857cf +Author: James Rodewig +Date: Tue Aug 13 08:24:53 2019 -0400 + + [DOCS] Reformats cat shards API (#45392) + +commit 0ff723852fc697f63ac1eca17674a2098684fc29 +Author: István Zoltán Szabó +Date: Tue Aug 13 14:22:36 2019 +0200 + + [DOCS] Reformats cluster update settings API (#45337) + +commit f4b001dc4b497be1946e3c757dc2db7d7944e3cb +Author: István Zoltán Szabó +Date: Tue Aug 13 13:53:43 2019 +0200 + + [DOCS] Adds heading between examples to better separate them. (#45499) + +commit 356a632b95d8b42a80e04d2a7bb7a7942a47336b +Author: István Zoltán Szabó +Date: Tue Aug 13 13:26:57 2019 +0200 + + [DOCS] Reformats cluster node info API (#45446) + + Co-Authored-By: James Rodewig + +commit 4ee7ac25aef6e110c38a130388b40c51061f6016 +Author: István Zoltán Szabó +Date: Tue Aug 13 12:46:47 2019 +0200 + + [DOCS] Reformats cluster node stats API (#45441) + + Co-Authored-By: James Rodewig + +commit dd527b4e915c5a476552220a4d2783d3892dfee5 +Author: Alexander Reelsen +Date: Tue Aug 13 12:15:23 2019 +0200 + + Fix watcher HttpClient URL creation (#45207) + + The http client could end up creating URLs, that did not resemble the + original one, when encoding. This fixes a couple of corner cases, where + too much or too few slashes were added to an URI. + + Closes #44970 + +commit 00e4fba2fb0ac124e5d166daba42be6da6022de4 +Author: Armin Braun +Date: Tue Aug 13 10:43:30 2019 +0200 + + Simplify and Optimize RestController Slightly (#45419) (#45485) + + * Simplify the path iterator to generate less garbage + * `dispatchRequest` always terminates, adjust code accordingly + +commit 0353eb929184fada0967ac609e4d1f00b86e5a17 +Author: Martijn van Groningen +Date: Tue Aug 13 09:11:04 2019 +0200 + + required changes after merging in upstream branch + +commit 488673f6362bf33b4f0b1062a8cf38c8b840fa27 +Author: Ryan Ernst +Date: Tue Aug 13 00:13:20 2019 -0700 + + Guard distro tests for pre oss in distribution filesnames (#45478) + + Before #45064, the bats tests skipped the upgrade tests when the random + upgrade version is before 6.3.0. This commit restores that behavior. + + closes #45476 + +commit 1951cdf1cb11ae1b8d8125995fdf17a489973da1 +Merge: 04626de6ae9 1aed388a24d +Author: Martijn van Groningen +Date: Tue Aug 13 09:12:31 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 1aed388a24d96430fc899fcff5a5987707957d48 +Author: Przemysław Witek +Date: Tue Aug 13 08:31:58 2019 +0200 + + Add view_index_metadata to roles.yml and remove as many df analytics test cases from build.gradle blacklist as possible. (#45451) (#45465) + +commit 471d940c44788bfcbd35600cbd7bb81151a1cd5c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Aug 13 09:06:18 2019 +1000 + + Refactor cluster privileges and cluster permission (#45265) (#45442) + + The current implementations make it difficult for + adding new privileges (example: a cluster privilege which is + more than cluster action-based and not exposed to the security + administrator). On the high level, we would like our cluster privilege + either: + - a named cluster privilege + This corresponds to `cluster` field from the role descriptor + - or a configurable cluster privilege + This corresponds to the `global` field from the role-descriptor and + allows a security administrator to configure them. + + Some of the responsibilities like the merging of action based cluster privileges + are now pushed at cluster permission level. How to implement the predicate + (using Automaton) is being now enforced by cluster permission. + + `ClusterPermission` helps in enforcing the cluster level access either by + performing checks against cluster action and optionally against a request. + It is a collection of one or more permission checks where if any of the checks + allow access then the permission allows access to a cluster action. + + Implementations of cluster privilege must be able to provide information + regarding the predicates to the cluster permission so that can be enforced. + This is enforced by making implementations of cluster privilege aware of + cluster permission builder and provide a way to specify how the permission is + to be built for a given privilege. + + This commit renames `ConditionalClusterPrivilege` to `ConfigurableClusterPrivilege`. + `ConfigurableClusterPrivilege` is a renderable cluster privilege exposed + as a `global` field in role descriptor. + + Other than this there is a requirement where we would want to know if a cluster + permission is implied by another cluster-permission (`has-privileges`). + This is helpful in addressing queries related to privileges for a user. + This is not just simply checking of cluster permissions since we do not + have access to runtime information (like request object). + This refactoring does not try to address those scenarios. + + Relates #44048 + +commit 97efb6a403f4ae69c113af17c12dd98691602dde +Author: Ryan Ernst +Date: Mon Aug 12 16:01:53 2019 -0700 + + Convert vagrant tests to per platform projects (#45064) + + The vagrant based tests currently reside in a single project, creating + dozens of tasks to manage starting and stopping the vagrant VM along + with running java and bats tests within each image. This all-in-one + pattern makes parallelizing packaging tests difficult. + + This commit rewrites the vagrant testing infrastructure to be + independent of the actual test runners, thus allowing each platform to + be handled in a separate subproject. Additionally, the java and bats + tests are changed to be run through a "destructive" gradle task, which + is run inside the VM. The combination of these will allow + parallelization both locally (through running several VMs at once) as + well as running the destructive tasks in CI machines dedicated to each + platform (thus removing the need for vagrant in CI). + +commit ae06a9399a01e36b75db6ba4deb4bded6e694ee9 +Author: Tim Brooks +Date: Mon Aug 12 17:44:54 2019 -0400 + + Fix bug in copying bytes for socket write (#45463) + + Currently we take the array of nio buffers from the netty channel + outbound buffer and copy their bytes to a direct buffer. In the process + we mutate the nio buffer positions. It seems like netty will continue to + reuse these buffers. This means than any data that is not flushed in a + call is lost. This commit fixes this by incrementing the positions after + the flush has completed. This is similar to the behavior that + SocketChannel would have provided and netty relied upon. + + Fixes #45444. + +commit dc1856ca539d6b057c72c8bba29c7378af20de80 +Author: Julie Tibshirani +Date: Mon Aug 12 14:10:32 2019 -0700 + + Make sure to validate the type before attempting to merge a new mapping. (#45157) + + Currently, when adding a new mapping, we attempt to parse + merge it before + checking whether its top-level document type matches the existing type. So when + a user attempts to introduce a new mapping type, we may give a confusing error + message around merging instead of complaining that it's not possible to add + more than one type ("Rejecting mapping update to [my-index] as the final + mapping would have more than 1 type..."). + + This PR moves the type validation to the start of + `MetaDataMappingService#applyRequest` so that we make sure the type matches + before performing any mapper merging. + + We already partially addressed this issue in #29316, but the tests there + focused on `MapperService` and did not catch this problem with end-to-end + mapping updates. + + Addresses #43012. + +commit 4d97d2c50fceff54bf0e2e5640cb37b5bc0358d1 +Author: Zachary Tong +Date: Mon Aug 12 17:17:17 2019 -0400 + + Revert "Only execute one final reduction in InternalAutoDateHistogram (#45359)" + + This reverts commit c0ea8a867e60ce223ddfe70e65ecfb42b36f0def. + +commit 2a1f0c7b4a62ea177925db906a0166a76c7555bc +Author: Mark Vieira +Date: Mon Aug 12 13:24:18 2019 -0700 + + Fix location of reaper jar on Windows (#45464) + + + (cherry picked from commit 5021c5d8bf815ed11a2fe9cf4d3a2c2ed2e2e0b3) + +commit 82d48cfcc9074a1f35cd44159f1174d63e1b32ed +Author: Chris Dean <53786748+wchrisdean@users.noreply.github.com> +Date: Mon Aug 12 15:13:11 2019 -0500 + + [DOCS] Added cross-link to snapshot lifecycle management. Closes #44588. (#45408) (#45468) + + merging #44588 changes into 7.x + +commit 8c4394d5d7f6d15ebba45f3d6fdc2d1d0ab2fb90 +Author: Julie Tibshirani +Date: Mon Aug 12 11:12:19 2019 -0700 + + Fix a bug where mappings are dropped from rollover requests. (#45411) + + We accidentally introduced this bug when adding a typeless version of the + rollover request. The bug is not present if include_type_name is set to true. + +commit a521e4c86f579ed9b195b2199cfbe80a4561110c +Author: Michael Basnight +Date: Mon Aug 12 13:32:25 2019 -0500 + + Retrieve processors instead of checking existence (#45354) + + The previous hasProcessors method would validate if a processor was + present within a pipeline, but would not return the contents of the + processors. This does not allow a consumer to inspect the processor for + specific metadata. The method now returns the list of processors based + on the class of the processor passed in. + +commit 472f6ef41abce06378721dd647f8fbabb9be4c95 +Author: Zachary Tong +Date: Mon Aug 12 14:45:08 2019 -0400 + + Mute InternalAutoDateHistogramTests#testReduceRandom() + +commit 04626de6ae91cac5a7696a89dc52811de552e332 +Author: Martijn van Groningen +Date: Mon Aug 12 20:36:10 2019 +0200 + + Add initial version of enrich processor docs. (#45084) + + Relates to #32789 + +commit c0ea8a867e60ce223ddfe70e65ecfb42b36f0def +Author: Zachary Tong +Date: Mon Aug 12 14:06:38 2019 -0400 + + Only execute one final reduction in InternalAutoDateHistogram (#45359) + + Because auto-date-histo can perform multiple reductions while + merging buckets, we need to ensure that the intermediate reductions + are done with a `finalReduce` set to false to prevent Pipeline aggs + from generating their output. + + Once all the buckets have been merged and the output is stable, + a mostly-noop reduction can be performed which will allow pipelines + to generate their output. + +commit 620cd4cb20059c541ae2925308b8f541e99b474f +Author: Ryan Ernst +Date: Mon Aug 12 09:28:44 2019 -0700 + + Fix old name of script context in docs (#45410) + + The docs contain an example how to limit the scripting contexts allowed, + but the example used an outdated name for the scoring context. + + closes #44232 + +commit 7e3379444bfc34d5c23f684082aa98ff2eb10540 +Author: Mark Vieira +Date: Mon Aug 12 09:17:55 2019 -0700 + + Fix build failure due to unknown task and disable test conventions + + (cherry picked from commit 8ed84bc5cef9bcfae6c817059f764d97e4451a4a) + +commit 250544c4eb32009b876498accdcbc2d13fc4f9ba +Author: Mark Vieira +Date: Mon Aug 12 09:10:26 2019 -0700 + + Upgrade to build scan plugin 2.4 (#45426) + +commit 421e9b8e8b3ee9ef1965280881054e9b6c3da705 +Author: Przemyslaw Gomulka +Date: Mon Aug 12 17:42:24 2019 +0200 + + Mute integ tests in native-multi-node-tests (#45457) + + Tracked at #45405 + +commit e3f618e1d32ff97d7be7381e065eb25d1f8a4312 +Author: James Rodewig +Date: Mon Aug 12 11:17:19 2019 -0400 + + [DOCS] Reformat query string query (#45296) + +commit bff4ef2d9593f639ef0a0328f0ec885ce89ce4eb +Author: James Rodewig +Date: Mon Aug 12 08:45:28 2019 -0400 + + [DOCS] Reformats cat templates API (#45403) + +commit c1c3bd011a0f4b6f1c0bf6e55e1453fbafdb3cd1 +Author: James Rodewig +Date: Mon Aug 12 08:40:33 2019 -0400 + + [DOCS] Reformats cat snapshots API (#45402) + +commit 7556d37925eed56ba85f566f81403f2675809848 +Author: James Rodewig +Date: Mon Aug 12 08:36:36 2019 -0400 + + [DOCS] Reformats cat thread pool API (#45385) + +commit e8acf6b34f11d0706fb55a7ddf3d2f17e6a1f4b5 +Author: James Rodewig +Date: Mon Aug 12 08:30:25 2019 -0400 + + [DOCS] Reformats cat repositories API (#45382) + +commit fb81346055594489f284cb6e2c6d84b4a7e22180 +Author: James Rodewig +Date: Mon Aug 12 08:25:09 2019 -0400 + + [DOCS] Reformats cat segments API (#45397) + +commit 4e79e04c0e818b26bb68c0d50f8d6adad369c5e9 +Author: James Rodewig +Date: Mon Aug 12 08:17:01 2019 -0400 + + [DOCS] Reformats cat recovery API (#45345) + +commit 499be8398fadba4c316f362171540b1c349776a5 +Author: Bogdan Pintea +Date: Mon Aug 12 13:42:08 2019 +0200 + + SQL: ODBC: document newest connection string parameters (#44185) + + * SQL: ODBC: document newest conn string parameters + + This commit adds the documentation for two newly added connection string + parameters: AutoEscapePVA and IndexIncludeFrozen. + + It also removes the recommended OSes from the prerequisites list and + places the recommendation distinctively: the unmet prerequisites will + fail the installation, while the driver would install on other OSes than + those recommended. + + * address review suggestions. + + - adjust phrasing for clearer message. + + (cherry picked from commit e18ac10c6e163a04f5b7cf7fa72f262882ffb711) + +commit d11ae08467f0af93f7b1ad9de74225dd4e963473 +Author: Przemyslaw Gomulka +Date: Mon Aug 12 11:01:32 2019 +0200 + + Muting ForecastIT.testOverflowToDisk (#45435) (#45438) + + awaits #45405 + +commit d02d6e40c2789530d86b112247cbaeef5261654b +Author: Dimitris Athanasiou +Date: Mon Aug 12 10:58:35 2019 +0300 + + [ML] Mute regression integ test + + Relates #45425 + +commit 2cb172f079e42868087913fce61f00a0add7c914 +Author: Albert Zaharovits +Date: Mon Aug 12 08:05:07 2019 +0300 + + CreateIndex and PutIndexTemplate with typeless mapping (#45120) + + This commit makes sure that mapping parameters to `CreateIndex` and + `PutIndexTemplate` are keyed by the type name. + + `IndexCreationTask` expects mappings to be keyed by the type name. + It asserts this for template mappings but not for the mappings in the request. + The `CreateIndexRequest` and `RestCreateIndexAction` mostly make it sure + that the mapping is keyed by a type name, but not always. + When building the create-index request outside of the REST handler, there are + a few methods to set the mapping for the request. Some of them add the type + name some of them do not. + For example, `CreateIndexRequest#mapping(String type, Map source)` + adds the type name, but + `CreateIndexRequest#mapping(String type, XContentBuilder source)` does not. + This PR asserts the type name in the request mapping inside `IndexCreationTask` + and makes all `CreateIndexRequest#mapping` methods add the type name. + +commit a9e14021898f463b91f839cc9d70b3389bf95d1a +Author: Armin Braun +Date: Mon Aug 12 05:14:45 2019 +0200 + + Remove Settings from BaseRestRequest Constructor (#45418) (#45429) + + * Resolving the todo, cleaning up the unused `settings` parameter + * Cleaning up some other minor dead code in affected classes + +commit f8c08c537bf1318285dc0396184e8a98fedf9e80 +Author: Emmanuel DEMEY +Date: Mon Aug 12 00:33:42 2019 +0200 + + Add snippet for the search_type query parameter (#43540) + +commit cf9a73b5accb7801e2b1fb58eea10f4f30f8b79b +Author: Nhat Nguyen +Date: Mon Aug 5 13:55:43 2019 -0400 + + Call afterWriteOperation after trim translog in peer recovery (#45182) + + testShouldFlushAfterPeerRecovery was added #28350 to make sure the + flushing loop triggered by afterWriteOperation eventually terminates. + This test relies on the fact that we call afterWriteOperation after + making changes in translog. In #44756, we roll a new generation in + RecoveryTarget#finalizeRecovery but do not call afterWriteOperation. + + Relates #28350 + Relates #45073 + +commit 25c61021011208b88eb0ff79d04c6e5c8159b35c +Author: Nhat Nguyen +Date: Sat Aug 3 16:17:23 2019 -0400 + + Trim local translog in peer recovery (#44756) + + Today, if an operation-based peer recovery occurs, we won't trim + translog but leave it as is. Some unacknowledged operations existing in + translog of that replica might suddenly reappear when it gets promoted. + With this change, we ensure trimming translog above the starting + sequence number of phase 2. This change can allow us to read translog + forward. + +commit 1cd464d675b3cc3fd478b5b530b2d3a316bc2c6b +Author: Armin Braun +Date: Sat Aug 10 10:21:01 2019 +0200 + + Isolate Request in Call-Chain for REST Request Handling (#45130) (#45417) + + * Follow up to #44949 + * Stop using a special code path for multi-line JSON and instead handle its detection like that of other XContent types when creating the request + * Only leave a single path that holds a reference to the full REST request + * In the next step we can move the copying of request content to happen before the actual request handling and make it conditional on the handler in question to stop copying bulk requests as suggested in #44564 + +commit 491880eddede016fbf9f6500112d07a2b04a2fbd +Author: Mark Vieira +Date: Fri Aug 9 21:09:16 2019 -0700 + + Fix build cache misses caused by embedded reaper jar (#45404) + + (cherry picked from commit 788feced760bc2a5f453ebb07f1dbb288c6232b2) + +commit b5c3c9767c04c6bd3fdff1d1aafebbce091e09db +Author: Mark Vieira +Date: Tue Aug 6 12:05:32 2019 -0700 + + Publish CI build scans to Gradle Enterprise (#45249) + + + (cherry picked from commit 957a232afc3ecd43326509043ab73cc6c2a26411) + +commit fac1a6f8e85d5ddc5fefb063e506edd148837726 +Author: Benjamin Trent +Date: Fri Aug 9 14:32:49 2019 -0500 + + [ML][Data Frame] have DataFrameTransformConfigUpdate#apply set Version (#45391) (#45400) + +commit bf4da6c6ad0c27c51bf89dc28d177341e671e62f +Author: Hendrik Muhs +Date: Fri Aug 9 20:30:11 2019 +0200 + + [ML-DataFrame] fix starting a batch data frame after stopping at runtime (#45340) (#45381) + + fix loading of next checkpoint after data frame transform has been stopped/started within one run + + closes #45339 + +commit 27497ff75f4cd94208a85af9ea5dece73b074a36 +Author: Dimitris Athanasiou +Date: Fri Aug 9 19:31:13 2019 +0300 + + [7.x][ML] Add regression analysis to DF analytics (#45292) (#45388) + + This commit adds a first draft of a regression analysis + to data frame analytics. There is high probability that + the exact syntax might change. + + This commit adds the new analysis type and its parameters as + well as appropriate validation. It also modifies the extractor + and the fields detector to be able to handle categorical fields + as regression analysis supports them. + +commit d1ed9bdbfd205dfcf597fdfbc102154410c079b3 +Author: Armin Braun +Date: Fri Aug 9 18:19:48 2019 +0200 + + Use StepListener to Simplify SnapshotResiliencyTests (#45233) (#45386) + + * Reduces complicated callback relations in `testSuccessfulSnapshotAndRestore` to flat steps of sequential actions + * Will refactor the other tests in this suit as a follow up + * This format certainly makes it easier to create more complicated tests that involve multiple subsequent snapshots as it would allow adding loops + +commit 9e6d874a4117dc95c5b517384e558161ccdefa89 +Author: Yannick Welsch +Date: Fri Aug 9 16:23:38 2019 +0200 + + Show BWC version in ClusterFormationFailureHelper (#45352) + + When having a cluster state from 6.x, display the metadata version as the cluster state version. + Avoids confusion where a cluster state from 6.x is displayed as version 0 even if has some actual + content. + +commit 846928a52a4ab6f14dccd5215041354efc927885 +Author: James Rodewig +Date: Fri Aug 9 08:53:25 2019 -0400 + + [DOCS] Reformats interval query (#45350) + +commit 1506d4436b13a4bd7853fb5b119e05c258e8d573 +Author: James Rodewig +Date: Fri Aug 9 08:48:19 2019 -0400 + + [DOCS] Reformats cat plugins API (#45344) + +commit eec87ffab802c9aba94faa88d89212dbbb2e237d +Author: James Rodewig +Date: Fri Aug 9 08:32:46 2019 -0400 + + [DOCS] Reformats simple query string query (#45343) + +commit 4ac25b23f623373becf63e51c595d09699e4fa1e +Author: Martijn van Groningen +Date: Fri Aug 9 12:34:48 2019 +0200 + + Add support for a more compact enrich values format (#45033) + + In the case that source and target are the same in `enrich_values` then + a string array can be specified. + + For example instead of this: + + ``` + PUT /_ingest/pipeline/my-pipeline + { + "processors": [ + { + "enrich" : { + "policy_name": "my-policy", + "enrich_values": [ + { + "source": "first_name", + "target": "first_name" + }, + { + "source": "last_name", + "target": "last_name" + }, + { + "source": "address", + "target": "address" + }, + { + "source": "city", + "target": "city" + }, + { + "source": "state", + "target": "state" + }, + { + "source": "zip", + "target": "zip" + } + ] + } + } + ] + } + ``` + This more compact format can be specified: + + ``` + PUT /_ingest/pipeline/my-pipeline + { + "processors": [ + { + "enrich" : { + "policy_name": "my-policy", + "targets": [ + "first_name", + "last_name", + "address", + "city", + "state", + "zip" + ] + } + } + ] + } + ``` + + And the `enrich_values` key has been renamed to `set_from`. + + Relates to #32789 + +commit 634a070430db42a58217b587280e5149c56ba91c +Author: Alpar Torok +Date: Fri Aug 9 13:32:27 2019 +0300 + + Restrict which tasks can use testclusters (#45198) + + * Restrict which tasks can use testclusters + + This PR fixes a problem between the interaction of test-clusters and + build cache. + Before this any task could have used a cluster without tracking it as + input. + With this change a new interface is introduced to track the tasks that + can use clusters and we do consider the cluster as input for all of + them. + +commit 5ddeb488a6fa46e5f9685de70d3c458ec31638b3 +Author: Yannick Welsch +Date: Fri Aug 9 09:01:15 2019 +0200 + + Allow _update on write alias (#45318) + + Using the document update API on aliases with a write index does not work. + + Follow-up to #31520 + +commit 5e1c0d598c5b9c9f0d1292f55c879373e00c00b5 +Author: Martijn van Groningen +Date: Fri Aug 9 09:12:03 2019 +0200 + + Added HLRC support for enrich put policy API. (#45183) + + This PR also adds HLRC docs. + + Relates to #32789 + +commit f1ee29f22ebf3455d5d04e4f865568ce99a40f1b +Author: Martijn van Groningen +Date: Fri Aug 9 08:55:38 2019 +0200 + + Added a custom api to perform the msearch more efficiently for enrich processor (#43965) + + Currently the msearch api is used to execute buffered search requests; + however the msearch api doesn't deal with search requests in an intelligent way. + It basically executes each search separately in a concurrent manner. + + This api reuses the msearch request and response classes and executes + the searches as one request in the node holding the enrich index shard. + Things like engine.searcher and query shard context are only created once. + Also there are less layers than executing a regular msearch request. This + results in an interesting speedup. + + Without this change, in a single node cluster, enriching documents + with a bulk size of 5000 items, the ingest time in each bulk response + varied from 174ms to 822ms. With this change the ingest time in each + bulk response varied from 54ms to 109ms. + + I think we should add a change like this based on this improvement in ingest time. + + However I do wonder if instead of doing this change, we should improve + the msearch api to execute more efficiently. That would be more complicated + then this change, because in this change the custom api can only search + enrich index shards and these are special because they always have a single + primary shard. If msearch api is to be improved then that should work for + any search request to any indices. Making the same optimization for + indices with more than 1 primary shard requires much more work. + + The current change is isolated in the enrich plugin and LOC / complexity + is small. So this good enough for now. + +commit 7d0aff0ed5483ae6920caabcfee033c3fe09176d +Author: Hendrik Muhs +Date: Fri Aug 9 09:03:47 2019 +0200 + + [ML-DataFrame] fix test failure in checkpoint retrieval (#45297) + + gracefully handle if index response returns null, increase and assert timeout + + closes #45238 + +commit a501d68f23511b89ed2c69da75ab98d3f162d4d4 +Author: Armin Braun +Date: Fri Aug 9 03:38:14 2019 +0200 + + Upgrade to Netty 4.1.38 (#45132) (#45364) + + * A number of fixes to buffer handling in the .37 and .38 -> we should stay up to date + +commit 2a99eaa7c2ba54bbd33ba905fab7542160af0481 +Author: Tal Levy +Date: Thu Aug 8 17:40:03 2019 -0700 + + Revert "removes the CellIdSource abstraction from geo-grid aggs (#45307) (#45353)" + + This reverts commit 7b0a8040de313b263505056baa70ec94fe338faf. + +commit 1794718e8e740895734bf263aa2c7df27cc6a722 +Author: Ryan Ernst +Date: Thu Aug 8 17:07:41 2019 -0700 + + Make git revision loading lazy (#45358) + + This commit makes the gitRevision property a lazy loaded value by + returning an Object implementing toString(). The Dockerfile template is + also changed to use groovy templates instead of the mavenfilter hack, so + converting to String will not happen until runtime. + +commit 12ed6dc9996b363ecf5a150872dbad1fe6d895d9 +Author: Armin Braun +Date: Fri Aug 9 01:56:32 2019 +0200 + + Only retain reasonable history for peer recoveries (#45208) (#45355) + + Today if a shard is not fully allocated we maintain a retention lease for a + lost peer for up to 12 hours, retaining all operations that occur in that time + period so that we can recover this replica using an operations-based recovery + if it returns. However it is not always reasonable to perform an + operations-based recovery on such a replica: if the replica is a very long way + behind the rest of the replication group then it can be much quicker to perform + a file-based recovery instead. + + This commit introduces a notion of "reasonable" recoveries. If an + operations-based recovery would involve copying only a small number of + operations, but the index is large, then an operations-based recovery is + reasonable; on the other hand if there are many operations to copy across and + the index itself is relatively small then it makes more sense to perform a + file-based recovery. We measure the size of the index by computing its number + of documents (including deleted documents) in all segments belonging to the + current safe commit, and compare this to the number of operations a lease is + retaining below the local checkpoint of the safe commit. We consider an + operations-based recovery to be reasonable iff it would involve replaying at + most 10% of the documents in the index. + + The mechanism for this feature is to expire peer-recovery retention leases + early if they are retaining so much history that an operations-based recovery + using that lease would be unreasonable. + + Relates #41536 + +commit 7b0a8040de313b263505056baa70ec94fe338faf +Author: Tal Levy +Date: Thu Aug 8 16:33:16 2019 -0700 + + removes the CellIdSource abstraction from geo-grid aggs (#45307) (#45353) + + CellIdSource is a helper ValuesSource that encodes GeoPoint + into a long-encoded representation of the grid bucket the point + is associated with. This complicates thing as usage evolves to + support shapes that are associated with more than one bucket ordinal. + +commit 68f91025508ec0e3040021465858a05f1d894044 +Author: Hendrik Muhs +Date: Thu Aug 8 12:22:30 2019 +0200 + + [ML-DataFrame] audit changes in the source index (#45282) + + add audits when the set of source indexes changes and in a special case runs empty + +commit 740d58fd469b95361ad60bb044c38e1fe343c4b7 +Author: Andrei Stefan +Date: Fri Aug 9 00:22:49 2019 +0300 + + SQL: Uniquely named inner_hits sections for each nested field condition (#45341) + + * Name each inner_hits section of nested queries differently and extract and combine the multiple values it generates into a single list. + This also introduces a limitation (its origin it's with Elasticsearch + though) on the sorting capabilities when the sorting is based on the + nested fields filtered: only one of the conditions applied to nested + documents will be used in the nested sorting. + + (cherry picked from commit cfc5cf68f6e83b07bb9006986d0903d6be418ec6) + +commit af908efa418339826d5e2469c5103f1b161dddae +Author: Tim Brooks +Date: Thu Aug 8 16:54:01 2019 -0400 + + Disable netty direct buffer pooling by default (#44837) + + Elasticsearch does not grant Netty reflection access to get Unsafe. The + only mechanism that currently exists to free direct buffers in a timely + manner is to use Unsafe. This leads to the occasional scenario, under + heavy network load, that direct byte buffers can slowly build up without + being freed. + + This commit disables Netty direct buffer pooling and moves to a strategy + of using a single thread-local direct buffer for interfacing with sockets. + This will reduce the memory usage from networking. Elasticsearch + currently derives very little value from direct buffer usage (TLS, + compression, Lucene, Elasticsearch handling, etc all use heap bytes). So + this seems like the correct trade-off until that changes. + +commit b19de550959df6630b560a1132b2cd44ba6f9675 +Author: Armin Braun +Date: Thu Aug 8 22:39:22 2019 +0200 + + Add missing wait to testAutomaticReleaseOfIndexBlock (#45342) (#45351) + + Today the test waits for one of the shards to be blocked, but this does not + mean that the block has been applied on all nodes, so a subsequent indexing + operation may still go through. + + Fixes #45338 + +commit d139896b669cdd148cfb39b73b62fa6a7533e0a8 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Aug 8 22:01:29 2019 +0200 + + Reindex share retry between hit sources (#44203) (#45348) + + The client and remote hit sources had each their own retry mechanism, + which would do the same. Supporting resiliency we would have to expand + on the retry mechanisms and as a preparation for that, the retry + mechanism is now shared such that each sub class is only responsible for + sending requests and converting responses/failures to common format. + + Part of #42612 + +commit 214cbb28df282b87e68e73213c79aae936ce6dae +Author: Mark Vieira +Date: Thu Aug 8 12:38:30 2019 -0700 + + Fix for build runtime classpath instability (#45347) + + (cherry picked from commit dee4ee2f0d4190ab54d0a4f0aa251d8c03e9db6d) + +commit a552b33276590adb756859990a4724788bbba2e0 +Author: Christoph Büscher +Date: Thu Aug 8 21:05:28 2019 +0200 + + Fix occasional SuggestSearchIT failure (#45330) + + Refreshes happening during indexing can result differen segment counts and + slightly skewed term statistics, which in turn has the potential to change + suggestion output slightly. In order to prevent this, disable refresh for the + affected tests. + + Closes #43261 + +commit 0ae103c40f671d26f43f45564a493ac9057def92 +Author: Mark Vieira +Date: Thu Aug 8 10:50:09 2019 -0700 + + Avoid unnecessary eager creation of Gradle tasks (#45098) (#45310) + +commit b716b840d3a989ce5b1d8e823888c3604bb4ae0f +Author: Jack Conradson +Date: Thu Aug 8 09:29:58 2019 -0700 + + Remove loop counter from Reserved in Painless AST. (#45298) + + This change adds a compiler pass to give each node the chance to store + settings necessary for analysis and writing. This removes the need to pass + this in a somewhat convoluted way through an additional class called + Reserved, and also removes the need to have the Walker set values for + settings on reserved. This is next step in decoupling the Painless grammar + from the Painless AST. + +commit 14545f8958d60930067c0e5fe1fc104de61db223 +Author: David Roberts +Date: Thu Aug 8 16:24:26 2019 +0100 + + [ML-DataFrame] Combine task_state and indexer_state in _stats (#45324) + + This commit replaces task_state and indexer_state in the + data frame _stats output with a single top level state + that combines the two. It is defined as: + + - failed if what's currently reported as task_state is failed + - stopped if there is no persistent task + - Otherwise what's currently reported as indexer_state + + Backport of #45276 + +commit bb429d3b5c5cebe4aea88c77918aa2ab3795bde3 +Author: Martijn van Groningen +Date: Thu Aug 8 16:30:08 2019 +0200 + + required changes after merge + +commit e53bb050dbd918d7995d3f98af6fe2b450696430 +Author: Dimitris Athanasiou +Date: Thu Aug 8 17:55:32 2019 +0300 + + Mute testAutomaticReleaseOfIndexBlock + + Relates #45338 + +commit 708f856940d7cb5104ff6b0e56cc610b27a32c79 +Merge: e3fd1e6c7d3 4e324708271 +Author: Martijn van Groningen +Date: Thu Aug 8 16:52:45 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 07c656fba9ef8e0372935b661c3bec80c10da6ef +Author: Andrey Ershov +Date: Thu Aug 8 17:25:21 2019 +0200 + + Mute testCustomDataPaths on Windows + + See #45333 + + (cherry picked from commit 671e1ad1068aee4b593ad0c8ab13ff60b4f125b8) + +commit f0f2294695053bffb66ed4ca59747627f8fca4be +Author: Mayya Sharipova +Date: Thu Aug 8 09:38:05 2019 -0400 + + Add filters in examples of vector functions (#45327) + +commit 86d65978905ef2c6ec61d832f6e23a9ccf746697 +Author: Zachary Tong +Date: Thu Aug 8 09:34:09 2019 -0400 + + Use newIndexSearcher() instead of newSearcher() (#45248) + + `newSearcher()` from lucene can randomly choose index readers which + are not compatible with our tests, like ParallelCompositeReader. + The `newIndexSearcher()` method on AggregatorTestCase is a wrapper + similar to newSearcher but compatible with our tests + +commit 4e3247082715e211fbc80e82c258ba7ffb8ec483 +Author: István Zoltán Szabó +Date: Thu Aug 8 15:26:08 2019 +0200 + + [DOCS] Reformats cluster reroute API. (#45328) + +commit 4d96c83854a51eaddecbaa89ad2b116d96ea562f +Author: István Zoltán Szabó +Date: Thu Aug 8 14:48:18 2019 +0200 + + [DOCS] Reformats cluster pending tasks API (#45280) + + Co-Authored-By: James Rodewig + +commit e72bda170304c331b2a03617f52626f3a9a12804 +Author: James Rodewig +Date: Thu Aug 8 08:37:22 2019 -0400 + + [DOCS] Reformats cat nodes API (#45285) + +commit b806e6edde4ad5954091c819d4844f0b0f22c01f +Author: James Rodewig +Date: Thu Aug 8 08:31:43 2019 -0400 + + [DOCS] Reformats cat pending tasks API (#45287) + +commit 276e9c66973e11af472da2e3bccac42a9d79b7d8 +Author: István Zoltán Szabó +Date: Thu Aug 8 13:43:55 2019 +0200 + + [DOCS] Adds supported time units ref to the ML and DF API params. (#45322) + +commit e0661330164dcb0c5de714843bc0a210b2c6e51f +Author: Martijn van Groningen +Date: Thu Aug 8 12:32:46 2019 +0200 + + Change the ingest simulate api to not include dropped documents (#44161) + + If documents are dropped by the `drop` processor then + these documents are returned as a `null` value in the response. + + === Example + + Create pipeline: + + ``` + PUT _ingest/pipeline/droppipeline + { + "processors": [ + { + "set": { + "field": "bla", + "value": "val" + } + }, + { + "drop": {} + } + ] + } + ``` + + Simulate request: + + POST _ingest/pipeline/droppipeline/_simulate + { + "docs": [ + { + "_source": { + "message": "text" + } + } + ] + } + + Response: + + ``` + { + "docs": [ + null + ] + } + ``` + + Response if verbose is enabled: + + ``` + { + "docs": [ + { + "processor_results": [ + { + "doc": { + "_index": "_index", + "_type": "_doc", + "_id": "_id", + "_source": { + "message": "text", + "bla": "val" + }, + "_ingest": { + "timestamp": "2019-07-10T11:07:10.758315Z" + } + } + }, + null + ] + } + ] + } + ``` + + Closes #36150 + + * Abort pipeline simulation in verbose mode when document has been dropped + by drop processor + +commit 99ddb8b3d8efd53c6795c7f4d9815a1a0bd0b3e2 +Author: Ioannis Kakavas +Date: Thu Aug 8 12:48:16 2019 +0300 + + Allow empty token endpoint for implicit flow (#45038) + + When using the implicit flow in OpenID Connect, the + op.token_endpoint_url should not be mandatory as there is no need + to contact the token endpoint of the OP. + +commit ddcc38cf1c29617d440bdc69683d37604ec014c3 +Author: David Turner +Date: Thu Aug 8 09:58:23 2019 +0100 + + More read-only-allow-delete docs (#45320) + + Adds to the `index.blocks.read_only_allow_delete` docs the information that + this block may be added or removed automatically, and rewords the + breaking-changes docs to mention the blocks explicitly and to recommend using a + different block. + + Relates #42559 + +commit e3fd1e6c7d304927ccd19f57e758c8601604690c +Author: Martijn van Groningen +Date: Thu Aug 8 10:17:20 2019 +0200 + + Add support for overwrite parameter in the enrich processor. (#45029) + + Similar to how it is supported in the set processor: + https://www.elastic.co/guide/en/elasticsearch/reference/current/set-processor.html + + Relates to #32789 + +commit 9f62c04637fe5402a15c9989ab41f3f4b00e0a12 +Author: István Zoltán Szabó +Date: Thu Aug 8 10:23:17 2019 +0200 + + [DOCS] Reformats cluster health and cluster state APIs (#45206) + + Co-Authored-By: James Rodewig + +commit fb959d188c74ae42614131457a8e13886613389e +Author: Martijn van Groningen +Date: Thu Aug 8 08:15:09 2019 +0200 + + Backport: Add description to force-merge tasks (#41365) (#45191) + + * Add description to force-merge tasks (#41365) + + This is static information that is part of the force merge request. + + Relates to #15975 + +commit fca458f1c86c85a83be231bfd13375efe9ecd680 +Author: Mark Vieira +Date: Wed Aug 7 13:14:09 2019 -0700 + + Use system properties for build cache configuration (#45295) + +commit e3599fded7887c2028bddbbc704c776febdd6119 +Author: Gordon Brown +Date: Wed Aug 7 14:11:05 2019 -0600 + + Add warning about versions to Deprecation API docs (#36624) + + Add a note that the Deprecation API may not be up to date with all + breaking changes until the last minor version in a major version series. + +commit 89861d0884711ad309e8cd787042300095f8242d +Author: Michael Basnight +Date: Wed Aug 7 10:52:14 2019 -0500 + + Add ingest processor existence helper method (#45156) + + This commit adds a helper method to the ingest service allowing it to + inspect a pipeline by id and verify the existence of a processor in the + pipeline. This work exposed a potential bug in that some processors + contain inner processors that are passed in at instantiation. These + processors needed a common way to expose their inner processors, so the + WrappingProcessor was created in order to expose the inner processor. + +commit 341ab48ec0501aeea9a16d6faca0e885882b34b0 +Author: Mark Vieira +Date: Wed Aug 7 09:04:20 2019 -0700 + + Improve SCM info in build scans (#45264) + +commit 5db9982f71cd54ea6503619744494d3451338cdf +Author: Benjamin Trent +Date: Wed Aug 7 10:37:35 2019 -0500 + + [7.x] [ML][Data Frame] Add update transform api endpoint (#45154) (#45279) + + * [ML][Data Frame] Add update transform api endpoint (#45154) + + This adds the ability to `_update` stored data frame transforms. All mutable fields are applied when the next checkpoint starts. The exception being `description`. + + This PR contains all that is necessary for this addition: + * HLRC + * Docs + * Server side + +commit 3a71b91dca7b934d91fbdde0f14973e9ff1516cf +Author: Benjamin Trent +Date: Wed Aug 7 10:37:09 2019 -0500 + + [ML][Data Frame] add support for geo_bounds aggregation (#44441) (#45281) + + This adds support for `geo_bounds` aggregation inside the `pivot.aggregations` configuration. + + The two points returned from the `geo_bounds` aggregation are transformed into `geo_shape` whose types are dynamic given the point's similarity. + + * `point` if the two points are identical + * `linestring` if the two points share either a latitude or longitude + * `polygon` if the two points are completely different + + The automatically deduced mapping for the resulting field is a `geo_shape`. + +commit 95d3a8e8add527788525cdcd744bef2972613713 +Author: István Zoltán Szabó +Date: Wed Aug 7 16:46:39 2019 +0200 + + [DOCS] Reformats cluster stats API and expands common params (#45270) + + Co-Authored-By: James Rodewig + +commit c7ec0b8431578b3982602c002de4b678c1361b06 +Author: Lee Hinman +Date: Wed Aug 7 08:01:42 2019 -0600 + + Include in-progress snapshot for a policy with get SLM policy… (#45245) + + This commit adds the "in_progress" key to the SLM get policy API, + returning a policy that looks like: + + ```json + { + "daily-snapshots" : { + "version" : 1, + "modified_date" : "2019-08-05T18:41:48.778Z", + "modified_date_millis" : 1565030508778, + "policy" : { + "name" : "", + "schedule" : "0 30 1 * * ?", + "repository" : "repo", + "config" : { + "indices" : [ + "foo-*", + "important" + ], + "ignore_unavailable" : true, + "include_global_state" : false + }, + "retention" : { + "expire_after" : "10m" + } + }, + "last_success" : { + "snapshot_name" : "production-snap-2019.08.05-oxctmnobqye3luim4uejhg", + "time_string" : "2019-08-05T18:42:23.257Z", + "time" : 1565030543257 + }, + "next_execution" : "2019-08-06T01:30:00.000Z", + "next_execution_millis" : 1565055000000, + "in_progress" : { + "name" : "production-snap-2019.08.05-oxctmnobqye3luim4uejhg", + "uuid" : "t8Idqt6JQxiZrzp0Vt7z6g", + "state" : "STARTED", + "start_time" : "2019-08-05T18:42:22.998Z", + "start_time_millis" : 1565030542998 + } + } + } + ``` + + These are only visible while the snapshot is being taken (or failed), + since it reads from the cluster state rather than from the repository + itself. + +commit 9384774b4c8daacc826477c65f79b912ec2f9849 +Author: István Zoltán Szabó +Date: Wed Aug 7 16:18:35 2019 +0200 + + [DOCS] Adds supported time units ref to the frequency and delay params. (#45283) + +commit 0ea00e486174bdbfe504106a91e2da6e80bb6d96 +Author: Alpar Torok +Date: Wed Aug 7 16:37:15 2019 +0300 + + Change how we pick bwc versions to check out (#45189) + + Prior to this PR we always checked out the latest bwc branches and had + an external mechanism to store the bwc versions used for every CI run so + we could both reproduce those builds and run additional tests using the + same combination. + + This adds complexities in setting up and maintaining CI and makes it + difficult to set up multi jobs. + + This change replaces that mechanism with a time based approach + that looks at the commit date of the current revision and picks the + newest on the bwc branch that's still older than that. + It also makes sure there are no merge commits in this interval. + + This new behavior will is ment to be enabled in CI only, for everything + except PR checks that will still use last available bwc revision. + +commit 46fc989ca2f98e0dc4420e473a088a8ffdb6d07b +Author: James Rodewig +Date: Wed Aug 7 09:31:07 2019 -0400 + + [DOCS] Reformats cat nodeattrs API (#45255) + +commit 5ade756275b0626f98875ca8b7eef2798cf7b062 +Author: James Rodewig +Date: Wed Aug 7 09:08:09 2019 -0400 + + [DOCS] Reformats cat indices API (#45239) + +commit d8423f0a80c1f84f656e69c9caa5c480b46d23f1 +Author: James Rodewig +Date: Wed Aug 7 09:03:31 2019 -0400 + + [DOCS] Reformat cat master API (#45240) + +commit 15d41a750842f1d39dcd4e13c766c2e9bd5e8033 +Author: Bas ten Berge +Date: Wed Aug 7 14:27:28 2019 +0200 + + [DOCS] Remove outdated 5.x community plugins (#40306) + + Our docs previously included several community plugins that are only supported for versions 5.x and earlier. This removes those plugins for our 6.6+ docs. + +commit be911e6a531e42632ed1942eb086f0210c5d6c52 +Author: Benjamin Trent +Date: Wed Aug 7 07:01:13 2019 -0500 + + [ML][Data Frames] Fix null aggregation handling in indexer (#45061) (#45257) + + * [ML][Data Frames] Fix null aggregation handling in indexer + + * addressing PR comments + + * adjusting error messages + +commit cd304c4def3e0622666746f9fe54355217131860 +Author: Bukhtawar +Date: Wed Aug 7 15:23:17 2019 +0530 + + Auto-release flood-stage write block (#42559) + + If a node exceeds the flood-stage disk watermark then we add a block to all of + its indices to prevent further writes as a last-ditch attempt to prevent the + node completely exhausting its disk space. However today this block remains in + place until manually removed, and this block is a source of confusion for users + who current have ample disk space and did not even realise they nearly ran out + at some point in the past. + + This commit changes our behaviour to automatically remove this block when a + node drops below the high watermark again. The expectation is that the high + watermark is some distance below the flood-stage watermark and therefore the + disk space problem is truly resolved. + + Fixes #39334 + +commit a869342910ccee00752d366d5dee2a23b3c065c0 +Author: Tanguy Leroux +Date: Wed Aug 7 10:35:53 2019 +0200 + + Restore DefaultShardOperationFailedException's reason after deserialization (#45203) + + The reason field of DefaultShardOperationFailedException is lost during serialization. + This is sad because this field is checked for nullity during xcontent generation and it + means that the cause won't be included in the generated xcontent and won't be + printed in two REST API responses (Close Index API and Indices Shard Stores API). + + This commit simply restores the reason from the cause during deserialization. + +commit bd59ee6c726e979175775fae0d620bf9050b4e0e +Author: Jason Tedor +Date: Tue Aug 6 21:14:42 2019 -0400 + + Fix clock used in update requests (#45262) + + We accidentally switched to using the relative time provider here. This + commit fixes this by switching to the appropriate absolute clock. + +commit 93181925788cad541dc12bb4aabdb565902e5a96 +Author: Julie Tibshirani +Date: Tue Aug 6 13:54:53 2019 -0700 + + Correct a code snippet in removal_of_types. (#45118) + + Previously, the reindex examples did not include `_doc` as the destination type. + This would result in the reindex failing with the error "Rejecting mapping + update to [users] as the final mapping would have more than 1 type: [_doc, + user]". + + Relates to #43100. + +commit a7a419bee87563826442aad74e36da7568a572d1 +Author: Tom Callahan +Date: Tue Aug 6 16:44:58 2019 -0400 + + Change Ldap SDK License to LGPL-2.1 (#45116) + + We currently use the unboundid ldap SDK, which is triply licensed under + GPL-2.0, LGPL-2.1, and the "UnboundID LDAP SDK Free Use License". We currently + identify the license as the latter, but LGPL-2.1 is the one we should be using + per our policy. + +commit 7c9c9a9cc4fb897a6b3a1a03528140c4151aefd9 +Author: Lisa Cawley +Date: Tue Aug 6 11:05:01 2019 -0700 + + [DOCS] Reformats ML update APIs (#45253) + +commit 51c1abc1125e4dca3c858e9ab6daab742e6e2cda +Author: James Rodewig +Date: Tue Aug 6 14:01:49 2019 -0400 + + [DOCS] Reformat match phrase prefix query (#45209) + +commit fc8a6fc9d057aa1f87883fec756bb9bb332cad07 +Author: Jack Conradson +Date: Tue Aug 6 10:04:14 2019 -0700 + + Decouple Painless AST Lambda generation from the grammar (#45111) + + This is the first step in decoupling the Painless AST from the grammar. The + Painless AST should be able to generate classes independently of how the + AST is generated from a grammar. (If I were to build a Painless AST by hand + in code this should be all that's necessary.) This change removes Lambda + name generation from the ANTLR grammar tree walker. It also removes + unnecessary node generation of new array function references from the + tree walker as well. + +commit 9a142ff25cd6ece44bdb5e35a5b1b631bf127d07 +Author: Jason Tedor +Date: Tue Aug 6 12:58:46 2019 -0400 + + Introduce formal node ML role (#45174) + + This commit builds on the ability for plugins to introduce new roles to + add a formal node ML role. + +commit bb22b2e45fa33dc504399d627c2d98db33afc18f +Author: Lisa Cawley +Date: Tue Aug 6 09:38:47 2019 -0700 + + [DOCS] Reformats get rollup jobs API (#45114) + +commit f5d1381e011265ffd467f0ae58214875f8784612 +Author: David Turner +Date: Tue Aug 6 17:21:27 2019 +0100 + + Remove always-true param from IndicesService#stats (#45231) + + Parameter `includePrevious` is always true, so this commit inlines it. + +commit 355713b9ca64985645d581d548b7a369b752e533 +Author: David Turner +Date: Tue Aug 6 17:01:49 2019 +0100 + + Improve slow logging in MasterService (#45241) + + Adds a tighter threshold for logging a warning about slowness in the + `MasterService` instead of relying on the cluster service's 30-second warning + threshold. This new threshold applies to the computation of the cluster state + update in isolation, so we get a warning if computing a new cluster state + update takes longer than 10 seconds even if it is subsequently applied quickly. + It also applies independently to the length of time it takes to notify the + cluster state tasks on completion of publication, in case any of these + notifications holds up the master thread for too long. + + Relates #45007 + Backport of #45086 + +commit bb7f46da62380d953cdc2f83a1aa230979c4ba51 +Author: Mark Vieira +Date: Tue Aug 6 09:00:25 2019 -0700 + + Avoid building docker images when running precommit task (#45211) + +commit 422aca9a5db768936e66581f62fd0bdb1bb69067 +Author: Zachary Tong +Date: Tue Aug 6 10:38:56 2019 -0400 + + Fix Rollup job creation to work with templates (#43943) + + The PutJob API accidentally used an "expert" API of CreateIndexRequest. + That API is semi-lenient to syntax; a type could be omitted and the + request would work as expected. But if a type was omitted it would + not merge with templates correctly, leading to index creation that + only has the template and not the requested mappings in the request. + + This commit refactors the PutJob API to: + + - Include the type name + - Use a less "expert" API in an attempt to future proof against errors + - Uses an XContentBuilder instead of string replacing, removes json template + +commit b09b87480165658c620d11e74307aef4c2d9b119 +Author: James Rodewig +Date: Tue Aug 6 10:33:30 2019 -0400 + + [DOCS] Reformat cat fielddata API (#45202) + +commit 5d7fafec14030f19ad4dc73f6a3a009c1b12aae7 +Author: Armin Braun +Date: Tue Aug 6 16:11:19 2019 +0200 + + Add Assertion to Ensure Retries in S3BlobContainer (#45224) (#45230) + + * We need a `markSupported` input stream to retry uploads + * Relates #45153 + +commit 772ce1f59988d38f7fec46367b4afa2072c2ba33 +Author: Tanguy Leroux +Date: Tue Aug 6 16:04:24 2019 +0200 + + Add deprecation warning for Force Merge API (#44903) + + This commit adds a deprecation warning in 7.x for the Force Merge API + when both only_expunge_deletes and max_num_segments are set in a request. + + Relates #44761 + +commit 210593d8e5ee1c0085cbe28e0451e546c46bc90b +Author: Alexander Reelsen +Date: Tue Aug 6 15:20:04 2019 +0200 + + Add back lowercase processor in docs (#45090) + + This got lost in a refactoring in 9137d92ca613b7fe40574789686575c4404089c1 + +commit 1c96e8a6af4dfe2123b9532f9719e75499760225 +Author: James Rodewig +Date: Tue Aug 6 08:40:52 2019 -0400 + + [DOCS] Reformat cat health API (#45218) + +commit 2489508e1ff8ea1b320fc6344bdd40d299014d7c +Author: James Rodewig +Date: Tue Aug 6 08:36:22 2019 -0400 + + [DOCS] Reformat cat count API (#45160) + +commit bc60eebfc81055417c4dfd0ac5b982ae35538ab0 +Author: Alpar Torok +Date: Tue Aug 6 15:33:39 2019 +0300 + + Add retries to vault access in init scripts + +commit 17846212bd1019bfd504472a7177d7f35ce7ffd1 +Author: Yannick Welsch +Date: Tue Aug 6 14:18:16 2019 +0200 + + Fix tests after backport of #44055 + +commit b3076adae63a60c14f51fc5ec337e447ecf487ee +Author: Alpar Torok +Date: Tue Aug 6 14:31:28 2019 +0300 + + Fix reaper build failures on Windows (#45205) + + We configure the service ID as the node's toString but this containes + characters that Windows doesn't like. + This PR fixes it by allowing only alphanumeric characters + +commit a453cd489e9563072da1048cbeec9b819d7bb3ef +Author: Yannick Welsch +Date: Tue Aug 6 12:25:38 2019 +0200 + + Run testExtendedSocketOptions only on JDK11+ (#44055) + + This functionality only works on JDK 11 or higher + +commit 5b1b146099364d8e3afac99844b930812255d35e +Author: Jason Tedor +Date: Tue Aug 6 06:02:28 2019 -0400 + + Normalize environment paths (#45179) + + This commit applies a normalization process to environment paths, both + in how they are stored internally, also their settings values. This + normalization is done via two means: + - we make the paths absolute + - we remove redundant name elements from the path (what Java calls + "normalization") + + This change ensures that when we compare and refer to these paths within + the system, we are using a common ground. For example, prior to the + change if the data path was relative, we would not compare it correctly + to paths from disk usage. This is because the paths in disk usage were + being made absolute. + +commit 7aeb2fe73cf1f17aee0d4c3c2b81231f6dd7546c +Author: Yannick Welsch +Date: Mon Aug 5 16:09:11 2019 +0200 + + Add per-socket keepalive options (#44055) + + Uses JDK 11's per-socket configuration of TCP keepalive (supported on Linux and Mac), see + https://bugs.openjdk.java.net/browse/JDK-8194298, and exposes these as transport settings. + By default, these options are disabled for now (i.e. fall-back to OS behavior), but we would like + to explore whether we can enable them by default, in particular to force keepalive configurations + that are better tuned for running ES. + +commit 81ea08b8ca37ebffe8b4c601d91f19a1247a0051 +Author: Alpar Torok +Date: Tue Aug 6 11:45:28 2019 +0300 + + Update ci bwc versions + +commit f91487baff5254d654708890d37266326f6f19ca +Author: David Turner +Date: Tue Aug 6 09:28:20 2019 +0100 + + Relax recovery duration in docs test (#45186) + + Sometimes the recovery in this docs test takes long enough that it is expressed + in `s` rather than `ms`. This commit relaxes the assertion to account for this. + +commit 6b5a2513a9b10971f334b2291d90578f4b7ec470 +Author: Hendrik Muhs +Date: Tue Aug 6 07:37:55 2019 +0200 + + [ML-DataFrame] introduce an abstraction for checkpointing (#44900) + + introduces an abstraction for how checkpointing and synchronization works, covering + + - retrieval of checkpoints + - check for updates + - retrieving stats information + +commit d0d808776f9e6028d9f851190435676ae786a76c +Author: Gordon Brown +Date: Mon Aug 5 17:28:59 2019 -0600 + + Add guide for using ILM with existing indices (#43667) + + In many cases, including migration from previous versions of data + shippers (e.g. Beats), it is useful to use ILM to manage historical + indices, which are no longer being written to. This commit adds a guide + which gives an example of how to do that. + +commit 87fb487299429604a5356aad2115581599cb748f +Author: Mark Vieira +Date: Mon Aug 5 15:52:46 2019 -0700 + + Fix failed dependency resolution with external build-tools users (#45107) (#45219) + +commit b5f88120b5b826f653efb70efeb58c81c2acb861 +Author: Igor Motov +Date: Fri Aug 2 14:34:02 2019 -0400 + + Geo: add Geometry-based query builders to QueryBuilders (#45058) + + Add Geometry-based method for creation of query builders in + QueryBuilder + + Relates to #44715 + +commit 3df1c76f9b88b025baba5bb495d931f0417c8530 +Author: Zachary Tong +Date: Mon Aug 5 12:15:42 2019 -0400 + + Allow pipeline aggs to select specific buckets from multi-bucket aggs (#44179) + + This adjusts the `buckets_path` parser so that pipeline aggs can + select specific buckets (via their bucket keys) instead of fetching + the entire set of buckets. This is useful for bucket_script in + particular, which might want specific buckets for calculations. + + It's possible to workaround this with `filter` aggs, but the workaround + is hacky and probably less performant. + + - Adjusts documentation + - Adds a barebones AggregatorTestCase for bucket_script + - Tweaks AggTestCase to use getMockScriptService() for reductions and + pipelines. Previously pipelines could just pass in a script service + for testing, but this didnt work for regular aggs. The new + getMockScriptService() method fixes that issue, but needs to be used + for pipelines too. This had a knock-on effect of touching MovFn, + AvgBucket and ScriptedMetric + +commit e5079ac2889b40d53e3679abb79d4c5f88aab983 +Author: Zachary Tong +Date: Mon Aug 5 11:56:52 2019 -0400 + + [7.x backport] Add more flexibility to MovingFunction window alignment (#45159) + + Introduce shift field to MovingFunction aggregation. + + By default, shift = 0. Behavior, in this case, is the same as before. + Increasing shift by 1 moves starting window position by 1 to the right. + + To simply include current bucket to the window, use shift = 1 + For center alignment (n/2 values before and after the current bucket), use shift = window / 2 + For right alignment (n values after the current bucket), use shift = window. + +commit adaa54c06636cb0be73a088f0239e41d1cf10c5d +Author: James Rodewig +Date: Mon Aug 5 11:17:37 2019 -0400 + + [DOCS] Reformats cat allocation API (#45158) + +commit f3570aa27b27378cfcc77890c13dbf4d760d28fb +Author: Alpar Torok +Date: Mon Aug 5 13:46:02 2019 +0300 + + CI specific init script updates (#45016) + + - Add a vault integration so that we don't need Jenkins to do that for + us + - This will make it easier to enable for windows too + - Move everything to a single file so we can read other secrets in the + same way + +commit 56083ba1ff499f598859fe0e6b53600f18b4b7a9 +Author: Nhat Nguyen +Date: Mon Aug 5 09:05:29 2019 -0400 + + Remove assertion after locally recover replica (#45181) + + If the disk becomes broken after we have locally recovered shard up to + the global checkpoint, then the assertion won't hold. + +commit 7bfaba98c283fe3ec624b53182a3c2b88ee94978 +Author: Benjamin Trent +Date: Mon Aug 5 09:12:11 2019 -0500 + + [ML][Data Frame] cleaning up and adjusting failure tests (#45101) (#45144) + +commit 2f4c342fff24aa37e9759af62e51df74618d84a4 +Author: James Rodewig +Date: Mon Aug 5 08:27:15 2019 -0400 + + [DOCS] Reformat cat alias API (#45119) + +commit dae648eb323e9f1b6a36b4ea987c1d8113fb5505 +Author: István Zoltán Szabó +Date: Mon Aug 5 13:28:22 2019 +0200 + + [DOCS] Makes clearer the note under freq_rare. (#45193) + +commit 13a167051f0840a928a488929eb336c45550b4cf +Author: David Turner +Date: Mon Aug 5 08:17:40 2019 +0100 + + Remove fileBasedRecovery flag (#45146) + + Today `RecoveryTarget#prepareForTranslogOperations` takes a boolean flag + indicating whether the recovery is file-based or not. This was used in 6.x to + bootstrap some commit data that were missing in indices created in 5.x: + + https://github.com/elastic/elasticsearch/blob/b506955f8d442c5c1659fd3840d8db14b3f969d1/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java#L298-L300 + + This flag no longer has any effect, so this commit removes it. + + Backport of #45131 to 7.x. + +commit 41815ed61476ef27b84abd3fb408bbb277d10e9f +Author: Armin Braun +Date: Mon Aug 5 07:22:42 2019 +0200 + + Optimize StreamInput#readString (#44930) (#45180) + + * Resolve TODO in `readString` by moving to reading chunks of `byte[]` instead of going byte by byte + * Motivated by `readString` showing up as a significant user of CPU time on the IO thread in Rally PMC benchmark + * Benchmarking this: + * Could not reproduce a slowdown in the potential worst case (one or two non-ascii chars) since in this case the cost of creating the string itself exceeds the read times anyway + * Speedup for 50%+ for reading 200 char ascii strings from `ByteBuf` or pages bytes backed streams + * Longer strings obviously get bigger speedups + * More ascii chars -> more speedup + +commit 89b21102e503b5e3ab508193814775b0664abffe +Author: Jason Tedor +Date: Sun Aug 4 19:21:39 2019 -0400 + + Upgrade to JDK 12.0.2 (#45172) + + This commit bumps the embedded JDK from 12.0.1 to 12.0.2. + +commit 872ae4d6c26645e6d395a74daef6de0d5f7ba5f2 +Author: Jason Tedor +Date: Sun Aug 4 13:50:26 2019 -0400 + + Add OCI annotations and adjust existing annotations (#45167) + + The org.label-schema labels on Docker images have been superseded by + pre-defined OCI annotations. However, there is still a lot of tooling in + use that relies on the org.label-schema, so we do not want to drop + them. This commit adds values for the org.opencontainers.image + pre-defined annotation keys. Additionally, we correct an issue with the + label used to represent the license, to use the org.label-schema.license + label. While this label was never accepted into the org.label-schema + specfication (because this specification was superseded, it's not that + it was explicitly rejected) there are containers out there using this + label. In particular, our base image is and so we need to override + otherwise we inherit, and end up mis-reporting the license. + +commit e708710f3857995130c50975217824fb5e5d6538 +Author: Jason Tedor +Date: Sun Aug 4 11:08:20 2019 -0400 + + Simplify handling of git revision (#45175) + + This commit simplifies the handling of git revision in the build. In + particular we remove pushing git revision through the generate build + info and print build info tasks as the git revision does not need to be + cached. + +commit 78aaf1b2d576db465d2f1dda26646a8d28d956ed +Author: Jason Tedor +Date: Sat Aug 3 14:48:14 2019 -0400 + + Lift build date to global build info (#45166) + + This commit adds the build date to global build info mainly so that it + is accessible as an extension property. + +commit d78ecd9c096fd7dad9cc12d487f7e4a8a8687868 +Author: Jason Tedor +Date: Sat Aug 3 11:24:47 2019 -0400 + + Use the full hash in build info (#45163) + + This commit switches to using the full hash to build into the JAR + manifest, which is used in node startup and the REST main action to + display the build hash. + +commit 925bd4faa11a5fbfefe064665b9e5747e26f3f7f +Author: Ryan Ernst +Date: Fri Aug 2 18:58:04 2019 -0700 + + Use reaper process instead of shutdown hooks for testclusters (#44927) + + Testclusters currently provides protection from clusters living past the + life of a build by adding a shutdown hook to java. While this works in + some cases, it does not cover all cases like where the daemon is killed + with SIGKILL. + + To handle these other cases, this commit replaces the shutdown hooks with + a separate process (one per build) that manages reaping external services + if gradle dies. + +commit 5dc22e29c634d0320a155ae923e5cc2a59f0dedf +Author: Jason Tedor +Date: Fri Aug 2 19:01:29 2019 -0400 + + Add build hash to global build info (#45162) + + This commit adds the commit hash to the global build info, and adds the + git revision as an extension. There are a couple motivations for this + change: + - the current mechanism of getting the build hash does not work with + git worktrees (because jgit does not understand them) + - a follow-up will want to use the git revision when building the + Docker images, so we want it available as an extension + - it allows us to simplify our usage around the build hash as we no + longer have to hack around silliness in the info-scm plugin + + A follow-up will also stop using the short hash in the product build, so + that we use the full hash there. We already know that short hashes in + our codebase do collide, so we should move to the full hash to avoid + this problem. + +commit 984ba82251ca21171bb274eabd29f82d43604c89 +Author: Tim Brooks +Date: Fri Aug 2 17:31:31 2019 -0400 + + Move nio channel initialization to event loop (#45155) + + Currently in the transport-nio work we connect and bind channels on the + a thread before the channel is registered with a selector. Additionally, + it is at this point that we set all the socket options. This commit + moves these operations onto the event-loop after the channel has been + registered with a selector. It attempts to set the socket options for a + non-server channel at registration time. If that fails, it will attempt + to set the options after the channel is connected. This should fix + #41071. + +commit ffbe047c32b87d3be36ffb2cf2fb54aaebbf12f4 +Author: Zachary Tong +Date: Fri Aug 2 15:16:04 2019 -0400 + + Revert "Add more flexibility to MovingFunction window alignment (#44360)" + + This reverts commit 1a58a487f0dd45a76f700abad0d3f9e7dd5eab37. + +commit 1a58a487f0dd45a76f700abad0d3f9e7dd5eab37 +Author: Nikita Glashenko +Date: Sat Aug 3 00:09:48 2019 +0500 + + Add more flexibility to MovingFunction window alignment (#44360) + + Introduce shift field to MovingFunction aggregation. + + By default, shift = 0. Behavior, in this case, is the same as before. + Increasing shift by 1 moves starting window position by 1 to the right. + + To simply include current bucket to the window, use shift = 1 + For center alignment (n/2 values before and after the current bucket), use shift = window / 2 + For right alignment (n values after the current bucket), use shift = window. + +commit 54552edaf626c0b7791edd2a721ee1db8ad1df66 +Author: Jack Conradson +Date: Fri Aug 2 11:52:23 2019 -0700 + + Whitelist randomUUID in Painless (#45148) + + This whitelists randomUUID with the understanding that it's possible for + /dev/random to cause blocking on *nix systems. Users that need + randomUUID should switch their random generator source to /dev/urandom + if this is a concern for them. + +commit 00235bbecda24fe82c3319492ba04da618da5943 +Author: Lisa Cawley +Date: Fri Aug 2 10:56:05 2019 -0700 + + [DOCS] Reformats the security APIs (#45124) + +commit 4b8ae97e9d383c3adaa6d722faba7edcc3babf6d +Author: James Rodewig +Date: Fri Aug 2 14:15:12 2019 -0400 + + [DOCS] Update relevance score cross-references (#45092) + +commit 8dd74dfe0b509954d00a54af8deeea32c4640968 +Author: James Rodewig +Date: Fri Aug 2 14:09:46 2019 -0400 + + Rename "indices APIs" to "index APIs" (#44863) + +commit b607148ae921a4d34e01301649378dc9a6698f78 +Author: Alison Goryachev +Date: Fri Aug 2 13:58:44 2019 -0400 + + [DOCS] Fix watcher email action docs (#44877) + +commit 09bd6c4692cde808dba8ddd3b3cda843cafc7953 +Author: Lisa Cawley +Date: Fri Aug 2 08:36:39 2019 -0700 + + [DOCS] Clarifies bucket span in overall buckets API (#45110) + +commit 47386d4f0ae85e35dd38832c5d83cc30da511b77 +Author: Christoph Büscher +Date: Fri Aug 2 16:43:23 2019 +0200 + + [Docs] Correct README example snippet (#45133) + + The example tries to create a new index but the create index API requires the + "settings" parameter to be there. + + Closes #45129 + +commit a1f0285f0e40f8aabb1a8e688b2607cfd6737a38 +Author: David Roberts +Date: Fri Aug 2 15:03:54 2019 +0100 + + [TEST] Only test US locale in day/month order test in FIPS JVM (#45141) + + In the FIPS JVM the JVM default locale seems to leak into places + where it should be overridden. This change skips assertions + in TimestampFormatFinderTests.testGuessIsDayFirstFromLocale + that may be impacted. + + Fixes #45140 + +commit 9ff320d967c1b551e80be0c922005f87ee7bd28b +Author: David Turner +Date: Fri Aug 2 15:00:43 2019 +0100 + + Use index for peer recovery instead of translog (#45137) + + Today we recover a replica by copying operations from the primary's translog. + However we also retain some historical operations in the index itself, as long + as soft-deletes are enabled. This commit adjusts peer recovery to use the + operations in the index for recovery rather than those in the translog, and + ensures that the replication group retains enough history for use in peer + recovery by means of retention leases. + + Reverts #38904 and #42211 + Relates #41536 + Backport of #45136 to 7.x. + +commit 3366726ad1c8ce47f8066e6bf5f6a6bbeb1dcf56 +Author: Christoph Büscher +Date: Fri Aug 2 14:34:22 2019 +0200 + + Enable reloading of synonym_graph filters (#45135) + + Reloading of synonym_graph filter doesn't work currently because the search time + AnalysisMode doesn't get propagated to the TokenFilterFactory emitted by the + graph filters getChainAwareTokenFilterFactory() method. This change fixes that. + + Closes #45127 + +commit f657f3914d36494f1de07b411e79b600322ec621 +Author: James Rodewig +Date: Fri Aug 2 08:42:33 2019 -0400 + + [DOCS] Create common parameters file for REST APIs (#45117) + +commit 6b71621afc3bd592c9ce33c8e6994b68ffa344c3 +Author: Tomas Della Vedova +Date: Fri Aug 2 11:39:52 2019 +0200 + + Updated slm API spec parameters and URL (#44797) (#45102) + +commit f617585dbde2d2ce617ed78cf9fd9cb22de8649f +Author: David Roberts +Date: Thu Aug 1 20:07:55 2019 +0100 + + [ML] Improve CSV header row detection in find_file_structure (#45099) + + When doing a fieldwise Levenshtein distance comparison + between CSV rows, this change ignores all fields that + have long values, not just the longest field. + + This approach works better for CSV formats that have + multiple freeform text fields rather than just a single + "message" field. + + Fixes #45047 + +commit 9450505d5b78d2652edd22407e25ff2efb4ce1e5 +Author: Armin Braun +Date: Fri Aug 2 07:31:38 2019 +0200 + + Stop Passing Around REST Request in Multiple Spots (#44949) (#45109) + + * Stop Passing Around REST Request in Multiple Spots + + * Motivated by #44564 + * We are currently passing the REST request object around to a large number of places. This works fine since we simply copy the full request content before we handle the rest itself which is needlessly hard on GC and heap. + * This PR removes a number of spots where the request is passed around needlessly. There are many more spots to optimize in follow-ups to this, but this one would already enable bypassing the request copying for some error paths in a follow up. + +commit e21d58541ad91be9e2d621de59de0e0756c72258 +Author: Tim Vernum +Date: Fri Aug 2 12:29:43 2019 +1000 + + Improve errors when TLS files cannot be read (#45122) + + This change improves the exception messages that are thrown when the + system cannot read TLS resources such as keystores, truststores, + certificates, keys or certificate-chains (CAs). + + This change specifically handles: + + - Files that do not exist + - Files that cannot be read due to file-system permissions + - Files that cannot be read due to the ES security-manager + + Backport of: #44787 + +commit 590777150f317ded5cee305cf6ae325cf1513cfd +Author: Tim Vernum +Date: Fri Aug 2 12:28:59 2019 +1000 + + Explicitly fail if a realm only exists in keystore (#45091) + + There are no realms that can be configured exclusively with secure + settings. Every realm that supports secure settings also requires one + or more non-secure settings. + However, sometimes a node will be configured with entries in the + keystore for which there is nothing in elasticsearch.yml - this may be + because the realm we removed from the yml, but not deleted from the + keystore, or it could be because there was a typo in the realm name + which has accidentially orphaned the keystore entry. + + In these cases the realm building would fail, but the error would not + always be clear or point to the root cause (orphaned keystore + entries). RealmSettings would act as though the realm existed, but + then fail because an incorrect combination of settings was provided. + + This change causes realm building to fail early, with an explicit + message about incorrect keystore entries. + + Backport of: #44471 + +commit ae5c01e2d240b0b6cb76c23ebcc26ed4d9b57b1c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Aug 2 09:29:46 2019 +1000 + + Do not use scroll when finding duplicate API key (#45026) + + When we create API key we check if the API key with the name + already exists. It searches with scroll enabled and this causes + the request to fail when creating large number of API keys in + parallel as it hits the number of open scroll limit (default 500). + We do not need the search context to be created so this commit + removes the scroll parameter from the search request for duplicate + API key. + +commit 8a6675b9949e18e50193ab9345ced7cd2384a7db +Author: Dimitris Athanasiou +Date: Fri Aug 2 00:19:48 2019 +0300 + + [7.x][ML] Check dest index is empty when starting DF analytics (#45094) (#45112) + + If one tries to start a DF analytics job that has already run, + the result will be that the task will fail after reindexing the + dest index from the source index. The results of the prior run + will be gone and the task state is not properly set to failed + with the failure reason. + + This commit improves the behavior in this scenario. First, we + set the task state to `failed` in a set of failures that were + missed. Second, a validation is added that if the destination + index exists, it must be empty. + +commit c13285a382986059ae9f56e229675685f9b30855 +Author: Mark Vieira +Date: Thu Aug 1 14:18:03 2019 -0700 + + Remove unnecessary plugin application and project configuration (#45100) + +commit 3f94e2ea43fbaa2e0e83edd31c83a37bfa38ae79 +Author: Jim Ferenczi +Date: Thu Aug 1 20:21:15 2019 +0200 + + Sparse role queries can throw an NPE (#45053) + + Sparse role queries are executed differently than other queries in order + to account for the fact that most of the documents are filtered from search. + However this special execution does not set the scorer for the query so any + collector that needs to access the score of a document fails with an NPE. + This change fixed this bug by setting the scorer before collecting any hits + when intersecting the main query and the sparse role. + +commit 5f50da947a3c13d0cb582bb086fc5f16bcf3dddb +Author: William Brafford +Date: Thu Aug 1 13:27:08 2019 -0400 + + Fix bug in the Settings#processSetting method (#45095) + + The Settings#processSetting method is intended to take a setting map and add a + setting to it, adjusting the keys as it goes in case of "conflicts" where the + new setting implies an object where there is currently a string, or vice + versa. processSetting was failing in two cases: adding a setting two levels + under a string, and adding a setting two levels under a string and four levels + under a map. This commit fixes the bug and adds test coverage for the + previously faulty edge cases. + + * fix issue #43791 about settings + * add unit test in testProcessSetting() + +commit 138865a58e3c7aa222a2ff586ae144028d63ed26 +Author: James Rodewig +Date: Thu Aug 1 12:06:34 2019 -0400 + + [DOCS] Reformat script score query (#45087) + +commit 7f74790db2061f5a9bc310f8818cc918fd372135 +Author: Lisa Cawley +Date: Thu Aug 1 09:12:33 2019 -0700 + + [DOCS] Adds 7.3.0 ml-cpp release notes (#45019) + +commit 4e00e119ed48c7814c02193b050ad140214ccb9a +Author: lcawl +Date: Thu Aug 1 09:10:15 2019 -0700 + + [DOCS] Fixes missing discrete attribute + +commit 917510d3e4b6758f333759667e9ed85c853f3c57 +Author: Yannick Welsch +Date: Thu Aug 1 17:29:31 2019 +0200 + + Always use primary term of operation in InternalEngine (#45083) + + We keep adding the current primary term to operations for which we do not assign a sequence + number. This does not make sense anymore as all operations which we care about have + sequence numbers now. The goal of this commit is to clean things up in InternalEngine and + reduce the complexity. + +commit 48dc53f8d2cf67a41efea6073a89f802b8c16711 +Author: Armin Braun +Date: Thu Aug 1 17:26:08 2019 +0200 + + Make PathTrieIterator a Little more Memory Efficient (#44951) (#45070) + + * There's no need to have the trie iterator hold another reference to the request object (which could be huge, see #44564) + * Also removed unused boolean field from trie node + +commit 187f1e776beb644313863ae19bc84daa6a7a3a09 +Author: James Rodewig +Date: Thu Aug 1 11:21:11 2019 -0400 + + [DOCS] Offset clone index API headings to correct nav (#45097) + +commit 3a487379c3f50fd7abd518463249679213d04938 +Author: Nhat Nguyen +Date: Thu Aug 1 08:28:17 2019 -0400 + + Tighten no pending scheduled refresh check (#45025) + + Previously, we use ThreadPoolStats to ensure that the scheduledRefresh + triggered by the internal refresh setting update is executed before we + index a new document. With that change (#40387), this test did not fail for + the last 3 months. However, using ThreadPoolStats is not entirely watertight + as both "active" and "queue" count can be 0 in a very small interval + when ThreadPoolExecutor pulls a task from the queue but before marking + the corresponding worker as active (i.e., lock it). + + Closes #39565 + +commit c088bafbbc75566e0418914195b0659c61c15a93 +Author: David Turner +Date: Thu Aug 1 13:45:52 2019 +0100 + + Wait for events in waitForRelocation (#45074) + + Adds a `waitForEvents(Priority.LANGUID)` to the cluster health request in + `ESIntegTestCase#waitForRelocation()` to deal with the case that this health + request returns successfully despite the fact that there is a pending reroute task which + will relocate another shard. + + Relates #44433 + Fixes #45003 + +commit 6c87845fc1b3d0312edcc8978f89e0020961a7e9 +Author: Przemysław Witek +Date: Thu Aug 1 14:36:59 2019 +0200 + + Persist DatafeedTimingStats with RefreshPolicy.NONE by default (#44940) (#45079) + +commit 2fc4b76ba8a29cb54444cd839483bfc890b11b02 +Author: Benjamin Trent +Date: Thu Aug 1 07:30:41 2019 -0500 + + [ML][Data Frame] adding breaking changes to doc (#45051) + + * [ML][Data Frame] adding breaking changes to doc + + * [DOCS] Edits data frame transform breaking change + + * Update migrate_7_3.asciidoc + +commit 532ade781628df7186f99100372164fe8190a6f4 +Author: David Turner +Date: Thu Aug 1 08:21:40 2019 +0100 + + More logging for slow cluster state application (#45007) + + Today the lag detector may remove nodes from the cluster if they fail to apply + a cluster state within a reasonable timeframe, but it is rather unclear from + the default logging that this has occurred and there is very little extra + information beyond the fact that the removed node was lagging. Moreover the + only forewarning that the lag detector might be invoked is a message indicating + that cluster state publication took unreasonably long, which does not contain + enough information to investigate the problem further. + + This commit adds a good deal more detail to make the issues of slow nodes more + prominent: + + - after 10 seconds (by default) we log an INFO message indicating that a + publication is still waiting for responses from some nodes, including the + identities of the problematic nodes. + + - when the publication times out after 30 seconds (by default) we log a WARN + message identifying the nodes that are still pending. + + - the lag detector logs a more detailed warning when a fatally-lagging node is + detected. + + - if applying a cluster state takes too long then the cluster applier service + logs a breakdown of all the tasks it ran as part of that process. + +commit b3be8f75f05cebb5b72b08f8087b07f06f1ef17a +Author: Hendrik Muhs +Date: Thu Aug 1 12:35:20 2019 +0200 + + Fix version logic after 7.3 release (BWC) (#45077) + + removes unreleased version 7.2.2 after release of 7.3.0 as it breaks the version verifier, add documentation that explains the logic + +commit a669efd2a4ec3b012a17716ccaac9fca1b60be60 +Author: Christoph Büscher +Date: Thu Aug 1 12:03:29 2019 +0200 + + Remove left-over AwaitsFix in RateClusterStateIT (#45043) + + Issues are closed and fixes in #42580 and #42430 seem to be merged to 7.x at + least. + +commit 39f280364b32f1915e7939c6e1345e30d7f82742 +Author: Martijn van Groningen +Date: Thu Aug 1 13:44:42 2019 +0700 + + required change after merging in 7.x branch + +commit aae2f0cff2428057add6be0108956d61d8cbb62f +Merge: db49cb505ef 9e901c5f94c +Author: Martijn van Groningen +Date: Thu Aug 1 13:38:03 2019 +0700 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 9e901c5f94ca43eb1fc86785dc64ef82a3d0f691 +Author: Lisa Cawley +Date: Wed Jul 31 15:49:27 2019 -0700 + + [DOCS] Edits 7.3 release highlights (#45066) + +commit fdc6c9853f19936f026dbfcdd06da61ab9e24132 +Author: Tim Brooks +Date: Wed Jul 17 10:28:17 2019 -0400 + + Do not write if connect incomplete (#44466) + + Currently, we do not handle READ or WRITE events until the channel + connection process is complete. However, the external write queue path + allows a write to be attempted when the conneciton is not complete. This + commit closes the loophole and only queues write operations when the + connection process is not complete. + +commit aff66e3ac5f3c75360030fb5538164b9fdf76b5f +Author: Tim Brooks +Date: Tue Jul 16 21:27:29 2019 -0400 + + Add Cors integration tests (#44361) + + This commit adds integration tests to ensure that the basic cors + functionality works for the netty and nio transports. + +commit 78c254a9c36fe1093d952036679367673457ae8c +Author: Lisa Cawley +Date: Wed Jul 31 11:47:10 2019 -0700 + + [DOCS] Adds notable 7.3 breaking changes (#45057) + +commit 3c4150cf725878ecff80341818ddc9610092a234 +Author: James Rodewig +Date: Wed Jul 31 14:31:28 2019 -0400 + + [DOCS] Reformat rank feature query. Add relevance score section. (#44975) + +commit 728b0cf9ff8c0cb354c25741d5c0f46c8ca248bc +Author: James Rodewig +Date: Wed Jul 31 14:18:22 2019 -0400 + + [DOCS] Update parameter format (#44703) + +commit f1bab2fa89454f6f2b0af6428fb85fab521e5e58 +Author: Flavio Pompermaier +Date: Wed Jul 31 20:10:05 2019 +0200 + + [DOCS] Correct sum_other_doc_count value in terms agg example (#45028) + + Closes issue #41902 + +commit 8d63bd1d1e955638b705f3a84dab91768186376d +Author: Armin Braun +Date: Wed Jul 31 18:55:31 2019 +0200 + + Cleanup Various Action- Listener and Runnable Usages (#42273) (#45052) + + * Dry up code for creating simple `ActionRunnable` a little + * Shorten some other code around `ActionListener` usage, in particular + when wrapping it in a `TransportResponseListener` + +commit ee663dc9ac7c54b64fc3189c68a6106b20bb24a5 +Author: Armin Braun +Date: Wed Jul 31 18:35:34 2019 +0200 + + Reenable Parallel Restore Test on Windows (#45037) (#45050) + + * As a result of #44096 this test shouldn't fail anymore on `master` and `7.4`+ so we should reenable it there + * For older versions we won't backport that change so the tests should stay disabled there + * Closes #44671 + +commit 0c687650884effd2a1c8eb7f21a00ba353d79f0e +Author: Mayya Sharipova +Date: Wed Jul 31 12:32:41 2019 -0400 + + Adds usage stats for vectors (#45023) + + Example of usage: + + _xpack/usage + + "vectors": { + "available": true, + "enabled": true, + "dense_vector_fields_count" : 1, + "sparse_vector_fields_count" : 1, + "dense_vector_dims_avg_count" : 100 + } + Backport for #44512 + +commit 5202d2624e0d717a97c288d036e6840dd600d78a +Author: Jack Conradson +Date: Wed Jul 31 08:21:46 2019 -0700 + + Add several context examples for Painless date documentation (#44985) + +commit 35291ae175180a62584517cd4f329a1772affd57 +Author: Christoph Büscher +Date: Wed Jul 31 17:17:21 2019 +0200 + + Remove muted AckIT and AckClusterUpdateSettingsIT (#45044) + + Reading up on #33673 it looks like parts of these tests have been reworked and + there is no intention to fix the remains on 7.x, so I think we can remove the + entire test. + +commit 8cc3c0dd93c9663d1583a6b6ea4bc1214caf713f +Author: Luca Cavanna +Date: Wed Jul 31 17:02:39 2019 +0200 + + Remove task null check in TransportAction (#45014) + + The task that TaskManager#register returns cannot be null. The method + enforces that it is not null after calling request#createTask. It is + then needless to check for null in the listener later. Also, added the + call to the delegate listener in a finally block, just to make sure. + +commit e85b53a955cac015113169ecb5f5bb2678fa477b +Author: Christoph Büscher +Date: Wed Jul 31 17:15:41 2019 +0200 + + Remove left-over AwaitsFix in DedicatedClusterSnapshotRestoreIT (#45042) + + The issue mentioned (#38845) seems to have been closed with #38891 so the test + can be re-activated. + +commit 6ed2a6e9dcb8b57ea97ca32a38d0617ad98a4fa4 +Author: Zachary Tong +Date: Wed Jul 31 09:17:12 2019 -0400 + + [DOCS] Add breaking change re: direct alias modification on followers + + Relates: #43017 + +commit c7d7230524390ddd76041237b3637c8967cdcac0 +Author: Armin Braun +Date: Wed Jul 31 17:11:34 2019 +0200 + + Stop Recreating Wrapped Handlers in RestController (#44964) (#45040) + + * We shouldn't be recreating wrapped REST handlers over and over for every request. We only use this hook in x-pack and the wrapper there does not have any per request state. + This is inefficient and could lead to some very unexpected memory behavior + => I made the logic create the wrapper on handler registration and adjusted the x-pack wrapper implementation to correctly forward the circuit breaker and content stream flags + +commit 56da35b70669256349184eb8456847b39ad63360 +Author: Ioannis Kakavas +Date: Wed Jul 31 17:54:00 2019 +0300 + + Indicate that some user APIs handle built-in users (#44857) + + The Get Users API also returns users form the restricted realm or built-in users, + as we call them in our docs. One can also change the passwords of built-in + users with the Change Password API + +commit c25f3dd5d0f61244cde033bf981ae8af06a9ff1d +Author: Zachary Tong +Date: Wed Jul 31 10:53:55 2019 -0400 + + Introduce 7.3.1 version (#45046) + +commit f6922bca2d3af160f91cbee45e6e26f38ce66103 +Author: Christoph Büscher +Date: Wed Jul 31 16:33:21 2019 +0200 + + Unmute test that seems to be fixed (#44432) + + Since #42509 is closed and the fix seems to have been backported to 7.x (#43539) + the test can be enabled again. + +commit c27ac3d24ceda58f8e4a92718bcc5bf938e20601 +Author: Andrey Ershov +Date: Wed Feb 13 16:54:18 2019 +0100 + + Unmute testClusterJoinDespiteOfPublishingIssues and testElectMasterWithLatestVersion (#38555) + + See my comments for #37539 and #37685 + + (cherry picked from commit 038d4ab2940340eca942e32b54044f183b7804d9) + +commit 5dd0e74e79e47e1e9ae7a47c5e9dfa2f8cd17f20 +Author: Daniel Mitterdorfer +Date: Wed Jul 31 13:13:39 2019 +0200 + + Clarify which circuit breaker settings are static (#44992) + + Most of the circuit breaker settings are dynamically configurable. + However, `indices.breaker.total.use_real_memory` is not. With this + commit we add a clarifying note that this specific setting is static. + + Closes #44974 + +commit f19f2adbe6824117a9ef6d817dca3501980eea7d +Author: Jesse Wright +Date: Wed Jul 31 06:35:42 2019 -0400 + + [Docs] Fix typo in rank-eval.asciidoc (#44978) + +commit 3c17d4379d9ff247c09670a8117950e362453c1e +Author: Tim Vernum +Date: Wed Jul 31 19:40:17 2019 +1000 + + Expand logging when SAML Audience condition fails (#45027) + + A mismatched configuration between the IdP and SP will often result in + SAML authentication attempts failing because the audience condition is + not met (because the IdP and SP disagree about the correct form of the + SP's Entity ID). + + Previously the error message in this case did not provide sufficient + information to resolve the issue because the IdP's expected audience + would be truncated if it exceeeded 32 characters. Since the error did + not provide both IDs in full, it was not possible to determine the + correct fix (in detail) based on the error alone. + + This change expands the message that is included in the thrown + exception, and also adds additional logging of every failed audience + condition, with diagnostics of the match failure. + + Backport of: #44334 + +commit 5e3010a60604b3073755ccb9aa7421329c108afa +Author: David Roberts +Date: Tue Jul 23 12:55:56 2019 +0100 + + Use system context for looking up connected nodes (#43991) + + When finding nodes in a connected cluster for cross cluster + search the requests to get cluster state on the connected + cluster should be made in the system context because + logically they are equivalent to checking a single detail + in the local cluster state and should not require that the + user who made the request that is using this method in its + implementation is authorized to view the entire cluster + state. + + Fixes #43974 + +commit 899c62ad02634bb6716ca7347755671563d969b2 +Author: Lisa Cawley +Date: Tue Jul 30 14:59:20 2019 -0700 + + [DOCS] Remove xpack terminology from installation pages (#44973) + +commit 1a1bb4707d6971a1b5154d3c155d379acc7fa3e1 +Author: Igor Motov +Date: Tue Jul 30 12:36:39 2019 -0400 + + Geo: move indexShape to AbstractGeometryFieldMapper.Indexer (#44979) + + Move indexShape functionality into AbstractGeometryFieldMapper to make + it more unit testable. + + Relates to #43644 + +commit e4b7ae211bae06f259b552328ad561afa707555e +Author: Lisa Cawley +Date: Tue Jul 30 10:52:23 2019 -0700 + + [DOCS] Updates terms in machine learning get APIs (#44986) + +commit a154b73d991166538496ca5e319fa5f9a4ca7154 +Author: Mayya Sharipova +Date: Tue Jul 30 14:23:05 2019 -0400 + + Assure index ops are successful for SimpleNestedIT (#44815) + + relates to #44486 + +commit 3f48720d4163e476efa219ba74e6d47d405df043 +Author: Benjamin Trent +Date: Tue Jul 30 13:05:07 2019 -0500 + + [ML][Data Frames] unify validation exceptions between PUT/_preview (#44983) (#45012) + + * [ML][Data Frames] unify validation exceptions between PUT/_preview + + * addressing PR comments + +commit 979d0a71c734a33d5cf7d4007c069ea1361e4bd0 +Author: Nhat Nguyen +Date: Tue Jul 30 10:15:37 2019 -0400 + + Remove leniency during replay translog in peer recovery (#44989) + + This change removes leniency in InternalEngine during replaying translog + in peer recovery. + +commit 41a99c9e4a99a886ba1ed34891a020262f189f24 +Author: Jake Landis +Date: Tue Jul 30 11:52:34 2019 -0500 + + introduce 7.2.2 as a version (#44371) + + * introduce 7.2.2 as a version + +commit 03fea1c503d9ba752084ba2abfa257b28ef08692 +Author: Jake Landis +Date: Tue Jul 30 11:48:41 2019 -0500 + + introduce 6.8.3 as a version (#44708) + +commit 22feedf2892a9600303326b9fedd0ec4255df01b +Author: Benjamin Trent +Date: Tue Jul 30 11:32:58 2019 -0500 + + [ML][Data Frame] add support for bucket_selector (#44718) (#45008) + +commit 548c767b6bff72e5263363095d7941f725d917aa +Author: Armin Braun +Date: Tue Jul 30 17:16:41 2019 +0200 + + S3 3rd Party Test Goal (#44799) (#45004) + + * Create S3 Third Party Test Task that Covers the S3 CLI Tool + * Adjust snapshot cli test tool tests to work with real S3 + * Build adjustment + * Clean up repo path before testing + * Dedup the logic for asserting path contents by using the correct utility method here that somehow became unused + +commit d0cbf0cc7ffb6389d49e31a3e22f9a17277dde00 +Author: David Kyle +Date: Tue Jul 30 15:53:23 2019 +0100 + + Mute WatcherRestIT 20_minimal_body + + Relates to https://github.com/elastic/elasticsearch/issues/43988 + +commit ac1107318396b7ebfee8e47d93db38aad8dcbd5c +Author: Armin Braun +Date: Tue Jul 30 15:25:52 2019 +0200 + + Optimize Netty Frame Decoding (#44664) (#45001) + + * We should not create a new wrapper object if there's no bytes in the `ByteBuf` + * We should not create a new wrapped `ByteBuf` if it can't contain a message anyway because it doesn't even have enough bytes for a header left + +commit 78aa6143a64d1ce59fb04d4db5dd560f60154f54 +Author: David Kyle +Date: Tue Jul 30 13:01:09 2019 +0100 + + Mute FilteringAllocationIT testTransientSettingsStillApplied + + Relates to https://github.com/elastic/elasticsearch/issues/45003 + +commit 19426f9cdfc939103260cca8c86ed26a460e6323 +Author: István Zoltán Szabó +Date: Tue Jul 30 14:22:14 2019 +0200 + + [DOCS] Adds allow no jobs param to the GET, GET stats and Close APIs (#44503) + +commit c1b569ed4b04ca226b634f0cbaf7245c92a117de +Author: Yannick Welsch +Date: Tue Jul 30 13:10:14 2019 +0200 + + Revert "Mute Zen1IT#testMixedClusterDisruption" + + This reverts commit cf78ca58e3f9c402982f039a9223042c90f5bc74. + +commit 55f1dd8da6b99030d78283560b58387889902095 +Author: David Turner +Date: Tue Jul 30 11:46:14 2019 +0100 + + Close nodes properly in Coordinator tests (#44967) + + Today closing a `ClusterNode` in an `AbstractCoordinatorTestCase` uses + `onNode()` so has no effect if the node is not in the current list of nodes. + It also discards the `Runnable` it creates without having run it, so has no + effect anyway. + + This commit makes these tests much stricter about properly closing the nodes + started during `Coordinator` tests, by tracking the persisted states that are + opened, and adds an assertion to catch the trappy requirement that the closing + node still belongs to the cluster. + +commit cf78ca58e3f9c402982f039a9223042c90f5bc74 +Author: David Kyle +Date: Tue Jul 30 11:32:14 2019 +0100 + + Mute Zen1IT#testMixedClusterDisruption + +commit e18e9fa8c5473da6f03987be77c1dee2db5345fb +Author: David Kyle +Date: Tue Jul 30 10:35:05 2019 +0100 + + Mute SnapshotLifecycleServiceTests#testPolicyCRUD + + Relates to https://github.com/elastic/elasticsearch/issues/44997 + +commit 43bd8f2ba0fca216deecdb9512366dca33a9d993 +Author: Jim Ferenczi +Date: Tue Jul 30 10:28:29 2019 +0200 + + Fix aggregators early termination with breadth-first mode (#44963) + + This commit fixes a bug when a deferred aggregator tries to early terminate the collection. In such case the CollectionTerminatedException is not caught and + the search fails on the shard. This change makes sure that we catch the exception in order to continue the deferred collection on the next leaf. + + Fixes #44909 + +commit 5a0bd696fc2688260e1289196affa41d94391b36 +Author: Andrey Ershov +Date: Tue Jul 30 11:02:08 2019 +0200 + + Snapshot tool S3 cleanup 7.x backport (#44575) + + Backport of #44551 + +commit 4495140d1f7513bfb19b929898be6a4731170e90 +Author: Armin Braun +Date: Tue Jul 30 10:30:01 2019 +0200 + + Release Pooled Buffers Earlier for HTTP Requests (#44952) (#44991) + + * We should release the buffers right after copying and not only do so after we did all the request handling on the copy + * Relates #44564 + +commit f575370e2f18042ba04bd51b42a1d3837a550b62 +Author: Tim Vernum +Date: Tue Jul 30 16:32:04 2019 +1000 + + Fix broken short-circuit in getUnlicensedRealms (#44937) + + The existing equals check was broken, and would always be false. + + The correct behaviour is to return "Collections.emptyList()" whenever + the the active(licensed)-realms equals the configured-realms. + + Backport of: #44399 + +commit 598c4e72f93927abe9febf3caba787491f792073 +Author: Lee Hinman +Date: Mon Jul 29 15:51:14 2019 -0600 + + [7.x] Rename indexlifecycle to ilm and snapshotlifecycle to sl… (#44977) + + * Rename indexlifecycle to ilm and snapshotlifecycle to slm (#44917) + + As a followup to #44725 and #44608, which renamed the packages within + the x-pack project, this renames the packages within the core x-pack + project. It also renames 'snapshotlifecycle' within the HLRC to slm. + + * Fix one more import + +commit 4813728783c2a8ddfa2c2b543fb4006cb6a8704e +Author: Nhat Nguyen +Date: Mon Jul 29 14:53:09 2019 -0400 + + Remove leniency in reset engine from translog (#44711) + + Replaying operations from the local translog must never fail as those + operations were processed successfully on the primary before and the + mapping is up to update already. This change removes leniency during + resetting engine from translog in IndexShard and InternalEngine. + +commit 1a21682ed036deb1f374a00c51e094fd7f904158 +Author: Jack Conradson +Date: Mon Jul 29 11:56:26 2019 -0700 + + Fix JodaCompatibleZonedDateTime casts in Painless (#44874) + + This is a temporary fix during the Joda to Java datetime transition. This will + implicitly cast a JodaCompatibleZonedDateTime to a ZonedDateTime for + both def and static types. This is necessary to insulate users from needing + to know about JodaCompatibleZonedDateTime explicitly. + +commit aef419c0b0eaa566e551ab41b5112e3fba57a6ad +Author: Dimitris Athanasiou +Date: Mon Jul 29 21:59:10 2019 +0300 + + [7.x][ML] Catch any error thrown while closing data frame analytics process (#44958) (#44968) + + In case closing the process throws an exception we should be catching + it no matter its type. The process may have terminated because of a + fatal error in which case closing the process will throw a server + error, not an `IOException`. If this happens we fail to mark the + persistent task as failed and the task gets in limbo. + +commit 3b514f0dae2d65e0c422b8ebdc9c1831bb532578 +Author: Benjamin Trent +Date: Mon Jul 29 13:06:56 2019 -0500 + + [ML] update Instant serialization (#44765) (#44954) + + * [ML] update Instant serialization + + * addressing PR comments + + * removing unused import + +commit 0914c04b8e395111ad0c8840beff1d1a230dbb1b +Author: Lisa Cawley +Date: Mon Jul 29 09:45:09 2019 -0700 + + [DOCS] Updates to data frame transforms release highlight (#44907) + +commit b6cef227a59800acceccbe61b417b8b4492e7bfc +Author: Igor Motov +Date: Mon Jul 29 11:46:53 2019 -0400 + + Geo: fix geo query decomposition (#44924) + + The recent refactoring introduced an issue where queries where not + going through the decomposition processing. + + Fixes #44891 + +commit 9dd527328aee607a0d52f769b798919bde9a0daa +Author: Dimitris Athanasiou +Date: Mon Jul 29 18:23:56 2019 +0300 + + [ML] Outlier detection should only fetch docs that have the analyzed … (#44944) (#44959) + + As data frame rows with missing values for analyzed fields are skipped, + we can be more efficient by including a query that only picks documents + that have values for all analyzed fields. Besides improving the number + of documents we go through, we also provide a more accurate measurement + of how many rows we need which reduces the memory requirements. + + This also adds an integration test that runs outlier detection on data + with missing fields. + +commit a3cc32da64cf20d08f76eceb27e257adac84f6df +Author: Luca Cavanna +Date: Mon Jul 29 16:25:39 2019 +0200 + + TaskListener#onFailure to accept Exception instead of Throwable (#44946) + + TaskListener accepts today Throwable in its onFailure method. Though + looking at where it is called (TransportAction), it can never be + notified of a Throwable. + + This commit changes the signature of TaskListener#onFailure so that it + accepts an `Exception` rather than a `Throwable` as second argument. + +commit 8653c338389e677fcf284b9692c2e541032f5351 +Author: Yannick Welsch +Date: Mon Jul 29 16:36:01 2019 +0200 + + Fix testBlockingIncomingRequests (#44939) + + Adapted test to take non-blocking nature into account. + +commit 245c9b79146d89a1e7bb5ab995eae0e23dd9fb06 +Author: Michał Perlak +Date: Fri Jul 26 21:40:27 2019 +0200 + + Optimize Min and Max BKD optimizations (#44315) + + MinAggregator - skip BKD optimization when no result found after 1024 lookups. + MaxAggregator - skip unnecessary conversions. + +commit 3839840b0c519a11f5bc603e70986b56876ee7a8 +Author: James Rodewig +Date: Mon Jul 29 09:59:09 2019 -0400 + + [DOCS] Reformat script query (#44882) + +commit 601521f4c130f19882e85c865815416f77b80114 +Author: James Rodewig +Date: Mon Jul 29 08:37:01 2019 -0400 + + [DOCS] Rewrite `prefix` query docs (#41955) + +commit 652f943f30b108ca4f79a06418d6c473cf11c58d +Author: James Rodewig +Date: Mon Jul 29 08:34:50 2019 -0400 + + [DOCS] Reformat distance feature query (#44916) + +commit ad129f79474312e3d9bea5da3354f5db7780e00f +Author: István Zoltán Szabó +Date: Mon Jul 29 14:06:17 2019 +0200 + + [DOCS] Amends REST API TOC. (#44948) + +commit 8f1a3ab70a657874a3d5f980733ab9fc6361edd3 +Author: Sandeep Kanabar +Date: Mon Jul 29 16:17:56 2019 +0530 + + [Docs] Update daterange-aggregation.asciidoc (#44730) + + Correcting the value to be the same as that specified for "missing". + +commit d05f12dadb67e2c06bb3b81d6cae9332717a4a71 +Author: David Kyle +Date: Mon Jul 29 10:44:44 2019 +0100 + + [ML] Close any opened pipes if there is an error connecting to the process (#44869) + +commit 24873dd3e382c7046f31e5f0f0fcb3cc752bc096 +Author: Yannick Welsch +Date: Mon Jul 29 11:34:48 2019 +0200 + + Do not block transport thread on startup (#44939) + + We currently block the transport thread on startup, which has caused test failures. I think this is + some kind of deadlock situation. I don't think we should even block a transport thread, and + there's also no need to do so. We can just reject requests as long we're not fully set up. Note + that the HTTP layer is only started much later (after we've completed full start up of the + transport layer), so that one should be completely unaffected by this. + + Closes #41745 + +commit f5efafd4d61da58eb6a367fa1d25742b1c9c8ed8 +Author: Armin Braun +Date: Mon Jul 29 10:38:35 2019 +0200 + + Cleanup Deadcode o.e.indices (#44931) (#44938) + + * none of this is used anywhere + +commit db49cb505ef5cb6a8b12eb7bc866ddc0bdd9b26a +Merge: 480af1ccf2f dc26521b0f5 +Author: Martijn van Groningen +Date: Mon Jul 29 14:45:10 2019 +0700 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 480af1ccf2f163efc2a04905b5b490da4181f7fe +Author: James Baiera +Date: Mon Jul 29 02:47:46 2019 -0400 + + Fix build errors (#44933) + + Add EnrichPlugin to test cases that update cluster state + +commit dc26521b0f538500feedc3877f60274ca7cff58c +Author: István Zoltán Szabó +Date: Mon Jul 29 08:03:11 2019 +0200 + + [DOCS] Updates to outlier detection release highlight (#44911) + +commit cfc8d17bb41f88e4cc3296e0bce4d567642f4158 +Author: Igor Motov +Date: Fri Jul 26 12:14:05 2019 -0400 + + Geo: refactor geo mapper and query builder (#44884) + + Refactors out the indexing and query generation logic out of the + mapper and query builder into a separate unit-testable classes. + +commit 1561ab5420b9f52c2b822a54a57008561b68aa3b +Author: Yannick Welsch +Date: Fri Jul 26 22:27:13 2019 +0200 + + Guard open connection call in RemoteClusterConnection (#44921) + + Fixes an issue where a call to openConnection was not properly guarded, allowing an exception + to bubble up to the uncaught exception handler, causing test failures. + + Closes #44912 + +commit a041d1eacfa53c5871af9656b1ae417a0da5de02 +Author: Lisa Cawley +Date: Fri Jul 26 11:07:01 2019 -0700 + + [DOCS] Updates anomaly detection terminology (#44888) + +commit cef375f88382db537e31af96cad63aa4a571b13a +Author: Lisa Cawley +Date: Fri Jul 26 10:47:03 2019 -0700 + + [DOCS] Updates terms in machine learning datafeed APIs (#44883) + +commit d4b2d2133994efe180bfcb9abe19c7ad9d0f2299 +Author: Gordon Brown +Date: Fri Jul 26 11:57:38 2019 -0400 + + Add option to filter ILM explain response (#44777) + + In order to make it easier to interpret the output of the ILM Explain + API, this commit adds two request parameters to that API: + + - `only_managed`, which causes the response to only contain indices + which have `index.lifecycle.name` set + - `only_errors`, which causes the response to contain only indices in an + ILM error state + + "Error state" is defined as either being in the `ERROR` step or having + `index.lifecycle.name` set to a policy that does not exist. + +commit 97177a3b4fb3f12dc73b601a9220b23d8a97484d +Author: Adrien Grand +Date: Fri Jul 26 17:46:35 2019 +0200 + + Update 7.3 release notes. + +commit e1b626b947b96a455817efafa942e05e8cc0910b +Author: Tanguy Leroux +Date: Fri Jul 26 16:58:39 2019 +0200 + + Ensure index is green in SimpleClusterStateIT.testIndicesOptions() (#44893) + + SimpleClusterStateIT testIndicesOptions failed in #44817 because it tries to close + an index at the beginning of the test. With random index settings, it is possible that + the index has a high number of shards (10) and replicas (1), which means that on + CI this index can take time to be fully allocated. + + The close index request can fail in the case where replicas are still recovering operations. + Thiscommit adds a simple ensureGreen() at the beginning of the test to be sure that all + replicas are started before trying to close the index. + + closes #44817 + +commit 1340ff19bc0e9c808548877f8770b894c1a82e2b +Author: Armin Braun +Date: Fri Jul 26 17:05:50 2019 +0200 + + Fix Test Failure in ScalingThreadPoolTests (#44898) (#44901) + + * Due to #44894 some constellations log a deprecation warning here now + * Fixed by checking for that + +commit 79121ea127fdc5cc2231f876c06994cf8f6f618b +Author: Przemysław Witek +Date: Fri Jul 26 15:56:34 2019 +0200 + + [7.x] Implement exponential average search time per hour statistics. (#44683) (#44897) + +commit cd7ba9f3028b9e21b2dafdd42d53ac4704b0a6da +Author: István Zoltán Szabó +Date: Fri Jul 26 11:39:59 2019 +0200 + + [DOCS] Amends data frame analytics resources, GET, and PUT API docs (#44806) + + This PR addresses the feedback in https://github.com/elastic/ml-team/issues/175#issuecomment-512215731. + + * Adds an example to `analyzed_fields` + * Includes `source` and `dest` objects inline in the resource page + * Lists `model_memory_limit` in the PUT API page + * Amends the `analysis` section in the resource page + * Removes Properties headings in subsections + +commit 8848fcfb2277f5b86b1fe7aa484f25745ad0a476 +Author: Tanguy Leroux +Date: Fri Jul 26 10:12:59 2019 +0200 + + Ensure cluster is stable in ShrinkIndexIT.testShrinkThenSplitWithFailedNode (#44860) + + The test ShrinkIndexIT.testShrinkThenSplitWithFailedNode sometimes fails + because the resize operation is not acknowledged (see #44736). This resize + operation creates a new index "splitagain" and it results in a cluster state + update (TransportResizeAction uses MetaDataCreateIndexService.createIndex() + to create the resized index). This cluster state update is expected to be + acknowledged by all nodes (see IndexCreationTask.onAllNodesAcked()) but + this is not always true: the data node that was just stopped in the test before + executing the resize operation might still be considered as a "faulty" node + (and not yet removed from the cluster nodes) by the FollowersChecker. The + cluster state is then acked on all nodes but one, and it results in a non + acknowledged resize operation. + + This commit adds an ensureStableCluster() check after stopping the node in + the test. The goal is to ensure that the data node has been correctly removed + from the cluster and that all nodes are fully connected to each before moving + forward with the resize operation. + + Closes #44736 + +commit 6ea2b5dec073ebcd873cf8cfb8b535367310bea5 +Author: Jason Tedor +Date: Fri Jul 26 17:06:10 2019 +0900 + + Deprecate setting processors to more than available (#44889) + + Today the processors setting is permitted to be set to more than the + number of processors available to the JVM. The processors setting + directly sizes the number of threads in the various thread pools, with + most of these sizes being a linear function in the number of + processors. It doesn't make any sense to set processors very high as the + overhead from context switching amongst all the threads will overwhelm, + and changing the setting does not control how many physical CPU + resources there are on which to schedule the additional threads. We have + to draw a line somewhere and this commit deprecates setting processors + to more than the number of available processors. This is the right place + to draw the line given the linear growth as a function of processors in + most of the thread pools, and that some are capped at the number of + available processors already. + +commit ac131f986bc64c4563fb5e00f9384b80033643c6 +Author: Ioannis Kakavas +Date: Fri Jul 26 09:37:45 2019 +0300 + + Document xpack.security.authc.saml.realm for Kibana (#44705) + + Since 7.3, it's possible to explicitly configure the SAML realm to + be used in Kibana's configuration. This in turn, eliminates the need + of properly setting `xpack.security.public.*` settings in Kibana + and largely simplifies relevant documentation. + This also changes `xpack.security.authProviders` to + `xpack.security.authc.providers` as the former was deprecated in + favor of the latter in 7.3 in Kibana + +commit 821f6f893b76f05d372e9d154b0add18ef3a6c65 +Author: Ignacio Vera +Date: Fri Jul 26 08:14:59 2019 +0200 + + Upgrade to Lucene 8.2.0 release (#44859) (#44892) + +commit d128188c28c1e1582b9f7eb2652520775d7cbfda +Author: Nhat Nguyen +Date: Fri Jul 19 10:57:14 2019 -0400 + + Return seq_no and primary_term in noop update (#44603) + + With this change, we will return primary_term and seq_no of the current + document if an update is detected as a noop. We already return the + version; hence we should also return seq_no and primary_term. + + Relates #42497 + +commit aebfdf1477207a56fc738cecb6ebce21894d6d6d +Author: Mark Vieira +Date: Thu Jul 25 15:33:51 2019 -0700 + + Fix issue with Gradle daemons hanging indefinitely on shutdown (#44867) (#44878) + +commit 03605169f3162e115a67e9aed9e9e6613a934688 +Author: lcawl +Date: Thu Jul 25 15:03:57 2019 -0700 + + [DOCS] Clarifies dataframe transform validations + +commit bd8470e738c5bf460963b7300e3c06bdeb023de8 +Author: Yannick Welsch +Date: Thu Jul 25 22:51:04 2019 +0200 + + Asynchronously connect to remote clusters (#44825) + + Refactors RemoteClusterConnection so that it no longer blockingly connects to remote clusters. + + Relates to #40150 + +commit 0ce841915c8faeb05eebf28aa1113757686c471b +Author: Yannick Welsch +Date: Thu Jul 25 20:17:51 2019 +0200 + + Add Clone Index API (#44267) + + Adds an API to clone an index. This is similar to the index split and shrink APIs, just with the + difference that the number of primary shards is kept the same. In case where the filesystem + provides hard-linking capabilities, this is a very cheap operation. + + Indexing cloning can be done by running `POST my_source_index/_clone/my_target_index` and it + supports the same options as the split and shrink APIs. + + Closes #44128 + +commit fda4db4fab741cb507681f5d42eec8d69a84ac55 +Author: James Baiera +Date: Thu Jul 25 15:28:40 2019 -0400 + + fixup! Merge branch '7.x' into enrich-7.x + +commit a89860160b3ba415d06014232865058d83147040 +Author: Mark Vieira +Date: Thu Jul 25 12:16:38 2019 -0700 + + Expose Elasticsearch API nullability information to Kotlin compiler. (#43912) (#44518) + + This change allows the Kotlin compiler to type check methods annotated with the + org.elasticsearch.common.Nullable annotation in Elasticsearch Java + APIs as described in: https://kotlinlang.org/docs/reference/java-interop.html#jsr-305-support. + + (cherry picked from commit 0d0485ad9cf10e16b75b862b023b42827c375599) + +commit 21971feae8bfda5afef0c6163cbb0f892fde775d +Author: Lisa Cawley +Date: Thu Jul 25 11:20:42 2019 -0700 + + [DOCS] Updates terms in machine learning calendar APIs (#44866) + +commit 8bb8543fdff7ab9141eaac11fed53cb4e6830739 +Author: Przemysław Witek +Date: Thu Jul 25 20:46:56 2019 +0200 + + Treat PostDataActionResponse.DataCounts.bucketCount as incremental rather than absolute (total). (#44803) (#44856) + +commit 03dd22b56c14fe1bd0c2c8a567ff38d522d329c4 +Author: Ryan Ernst +Date: Thu Jul 25 11:40:21 2019 -0700 + + Add missing ZonedDateTime methods for joda compat layer (#44829) + + While joda no longer exists in the apis for 7.x, the compatibility layer + still exists with helper methods mimicking the behavior of joda for + ZonedDateTime objects returned for date fields in scripts. This layer + was originally intended to be removed in 7.0, but is now likely to exist + for the lifetime of 7.x. + + This commit adds missing methods from ChronoZonedDateTime to the compat + class. These methods were not part of joda, but are needed to act like a + real ZonedDateTime. + + relates #44411 + +commit c9909b09b54161c80757b5d474f8236994b2a4ef +Author: Lisa Cawley +Date: Thu Jul 25 11:32:31 2019 -0700 + + [DOCS] Adds command reference for elasticsearch-croneval (#43946) + +commit acb7f599a37549e4a9ede68284e9a06e20ab1899 +Author: Julie Tibshirani +Date: Thu Jul 25 10:34:37 2019 -0700 + + Fix an NPE when requesting inner hits and _source is disabled. (#44836) + + This PR makes two changes to FetchSourceSubPhase when _source is disabled and + we're in a nested context: + * If no source filters are provided, return early to avoid an NPE. + * If there are source filters, make sure to throw an exception. + + The behavior was chosen to match what currently happens in a non-nested context. + +commit c5528a25e6f58108a86c9d6095ddbfedfc04a214 +Merge: c357f81aa79 383d7b77135 +Author: James Baiera +Date: Thu Jul 25 13:12:56 2019 -0400 + + Merge branch '7.x' into enrich-7.x + +commit 6a60fd6d308b082836c1091a7344d03d26ebe36c +Author: lcawl +Date: Thu Jul 25 10:08:38 2019 -0700 + + [DOCS] Fixes terms in HLRC data frame transform APIs (#44838) (#44865) + +commit 48757da6e105cc51db595b7ac97ab9eac13b61cc +Author: Nicholas Knize +Date: Thu Jul 25 10:29:47 2019 -0500 + + [GEO] Fix GeoShapeQueryBuilder to check for valid spatial relations + + Refactor left out the spatial strategy check in GeoShapeQueryBuilder.relation + setter method. This commit adds that check back in. + +commit 133f848e9f514a8c87b9cfa090c934efa4550a51 +Author: Nick Knize +Date: Wed Jul 24 15:41:31 2019 -0500 + + [Geo] Refactor GeoShapeQueryBuilder to derive from AbstractGeometryQueryBuilder (#44780) + + Refactors GeoShapeQueryBuilder to derive from a new AbstractGeometryQueryBuilder that provides common parsing and build logic for spatial geometries. This will allow development of custom geometry queries by extending AbstractGeometryQueryBuilder preventing duplication of common spatial query logic. + +commit a79adca7e3774058b83d11a3ef7d6885c5c6ffd0 +Author: Lisa Cawley +Date: Thu Jul 25 08:58:16 2019 -0700 + + [DOCS] Updates terms in anomaly detection job APIs (#44839) + +commit 4a31c426e6f97ac39964ae997e7d02e99881c466 +Author: István Zoltán Szabó +Date: Thu Jul 25 16:45:06 2019 +0200 + + [DOCS] Adds allow no datafeeds query param to the GET, GET stats and STOP datafeed APIs (#44499) + +commit 2ca630645238ac4a1d402ce294358189602cc2db +Author: Hendrik Muhs +Date: Thu Jul 25 16:37:36 2019 +0200 + + do not assert on indexer state (#44854) + + remove the unreliable check for the state change + + fixes #44813 + +commit b2e969f4baf1b4579233dcd2bdd0f2a6c3d77076 +Author: David Roberts +Date: Thu Jul 25 15:19:32 2019 +0100 + + [ML-DataFrame] Remove ID field from data frame indexer stats (#44848) + + This is a followup to #44350. The indexer stats used to + be persisted standalone, but now are only persisted as + part of a state-and-stats document. During the review + of #44350 it was decided that we'll stick with this + design, so there will never be a need for an indexer + stats object to store its transform ID as it is stored + on the enclosing document. This PR removes the indexer + stats document ID. + + Backport of #44768 + +commit af937b14ae665a8230a2a7f14377a0e86b9ff757 +Author: Albert Zaharovits +Date: Thu Jul 25 16:52:21 2019 +0300 + + SecurityIndexManager handle RuntimeEx while reading mapping (#44409) + + Fixes exception handling while reading and parsing `.security-*` + mappings and templates. + +commit 383d7b77135ce1665b381ca59806e0a85618169e +Author: Armin Braun +Date: Thu Jul 25 10:50:04 2019 +0200 + + Cleanup Dead Code in Index Creation (#44784) (#44822) + + * Cleanup Dead Code in Index Creation + * This is all unused and the state of a create request is always `OPEN` + +commit 53f409e5ae2dc8ad8a5241f0284b2ff66006fdc1 +Author: Przemysław Witek +Date: Thu Jul 25 10:11:55 2019 +0200 + + Add result_type field to TimingStats and DatafeedTimingStats documents (#44812) (#44841) + +commit e0d4544ef63977c2dd3bad3c1862c81d568b9736 +Author: Yannick Welsch +Date: Wed Jul 24 17:43:18 2019 +0200 + + Close connection manager on current thread in RemoteClusterConnection (#44805) + + The problem is that RemoteClusterConnection closes the connection manager asynchronously, which races with the threadpool being shutdown at the end of the test. + + Closes #44339 + Closes #44610 + +commit 9944e193f97c6f9a1ef7c1df1aff1429683356d5 +Author: Tanguy Leroux +Date: Thu Jul 25 09:33:57 2019 +0200 + + [7.x] Clean up ShardFollowTasks for deleted indices (#44702) (#44790) + + Deleting a follower index does not delete its ShardFollowTasks, potentially + leaving many persistent tasks in the cluster that cannot be allocated on + nodes and unnecessary fill the logs. This commit adds a cluster state listener + (ShardFollowTaskCleaner) that completes (with a failure) any persistent task + that refers to a non existent follower index. + + I think that this bug has been introduced by #34404: before this change the + task would have been completed as failed and removed from the cluster state. + + Backport of #44702 and #44801 on 7.x + +commit fd74b63602270e2117392543517bb635dfb3f22a +Author: Andrei Stefan +Date: Thu Jul 25 10:26:38 2019 +0300 + + SQL: fix URI path being lost in case of hosted ES scenario (#44776) + + (cherry picked from commit 06dea859e8fddada868941aaae15e83b4f64babe) + +commit ee53f7e161d548f6150eee04f4a3d8c207dafabc +Author: Andrei Stefan +Date: Thu Jul 25 09:54:04 2019 +0300 + + SQL: [Tests] Re-enable testDriverConfigurationWithSSLInURL test with more logging (#44800) + + (cherry picked from commit 5b9ccd72e9a3bb65c8b7b06979a75cb795c17111) + +commit 2633d11eb751a5b3b2438ca9dc5eda7482bf100d +Author: Andrei Stefan +Date: Thu Jul 25 10:02:41 2019 +0300 + + Switch from using docvalue_fields to extracting values from _source (#44062) (#44804) + + * Switch from using docvalue_fields to extracting values from _source + where applicable. Doing this means parsing the _source and handling the + numbers parsing just like Elasticsearch is doing it when it's indexing + a document. + * This also introduces a minor limitation: aliases type of fields that + are NOT part of a tree of sub-fields will not be able to be retrieved + anymore. field_caps API doesn't shed any light into a field being an + alias or not and at _source parsing time there is no way to know if a + root field is an alias or not. Fields of the type "a.b.c.alias" can be + extracted from docvalue_fields, only if the field they point to can be + extracted from docvalue_fields. Also, not all fields in a hierarchy of + fields can be evaluated to being an alias. + + (cherry picked from commit 8bf8a055e38f00df5f49c8d97f632f69d6e00c2c) + +commit f9943a3e531ff382513fadae6c5ff738b5634dbb +Author: Igor Motov +Date: Wed Jul 24 14:07:10 2019 -0400 + + Geo: deprecate ShapeBuilder in QueryBuilders (#44715) + + Removes unnecessary now timeline decompositions from shape builders + and deprecates ShapeBuilders in QueryBuilder in favor of libs/geo + shapes. + + Relates to #40908 + +commit b3d8b39f2934e2df76bc22d7d3326ec7657f3eb5 +Author: Ryan Ernst +Date: Wed Jul 24 11:20:28 2019 -0700 + + Convert logging related gradle classes to java (#44771) + + This commit converts the logging related classes (only used for vagrant) + to java from groovy. + + relates #34459 + +commit 8bac13d5b2927fd06208400b1dc08d3396d82c75 +Author: Lisa Cawley +Date: Wed Jul 24 11:09:06 2019 -0700 + + [DOCS] Adds descriptions for put and start data frame transforms (#44724) + +commit a12be619f64f35adff4e17e590cb34821e335732 +Author: Enrico Zimuel +Date: Wed Jul 24 18:04:16 2019 +0200 + + Fix URL documentation in API specs (#44487) + +commit fde6fcdfac6a34a39a83732d70fb3b561eac9d80 +Author: Lee Hinman +Date: Wed Jul 24 11:27:35 2019 -0600 + + Order ILM actions in policy definition documentation (#44773) + + We already have a note that the order of actions is up to ILM for each + phase, this commit puts the actions in the same order as they will be + executed. + + Resolves #41729 + +commit 2ac54da60a91a898e0e488226709477ed9eaa5c4 +Author: Gordon Brown +Date: Wed Jul 24 11:38:15 2019 -0400 + + Fix swapped variables in error message (#44300) + + The alias name and index were in the incorrect order in this error + message. This commit corrects the order. + +commit 4cfd2fc6b2133822b2331bd99b69f2411a790490 +Author: David Turner +Date: Wed Jul 24 16:09:51 2019 +0100 + + Fix testFirstListElementsToCommaDelimitedStringReportsFirstElementsIfLong (#44785) + + This test can fail (super-rarely) if it generates a list of length 11 + containing a duplicate, because the `.distinct()` reduces the list length to 10 + and then it is not abbreviated any more. This change generalises the test to + cover lists of any random length. + +commit d8be9244f970f74cc6eff9c77872369e057311ee +Author: Armin Braun +Date: Wed Jul 24 16:03:37 2019 +0200 + + Fix Repository Cleanup Test Correctness (#44738) (#44751) + + * The tests were creating the corruption and asserting its existence not on the repository base path but on a clean path. + As a result the consistency assertion on the repository wouldn't see the corruption ever an pass even if the cleanup was broken for repositories that have a non-root base path + +commit a8905ef14284e4ba32200135911508a7f06585a9 +Author: Tanguy Leroux +Date: Wed Jul 24 15:51:01 2019 +0200 + + [7.x] Add CloseIndexResponse to HLRC (#44349) (#44788) + + The CloseIndexResponse was improved in #39687; this commit + exposes it in the HLRC. + + Backport of #44349 to 7.x. + +commit 04cb3aebd507d8edd40ce64b46dda187031f235f +Author: Andrei Stefan +Date: Wed Jul 24 15:55:20 2019 +0300 + + Use hasValue() methods from aggregations' InspectionHelpers (#44745) + + Use InspectionHelper classes to decide if the aggregations should return null (in case there is no value) or the value itself. + + (cherry picked from commit dafd7b039b0da072750e8f57e7572d24f7aad44a) + +commit 0418eb35a7db38280d180a6012c1f100cf54d1e4 +Author: Adrien Grand +Date: Wed Jul 24 15:05:23 2019 +0200 + + Add 7.3 release highlights. (#44153) + + Add 7.3 release highlights. + +commit ad7c164dd0112beb8eb59f2b2d19b33da9375823 +Author: James Rodewig +Date: Wed Jul 24 08:37:37 2019 -0400 + + [DOCS] Rewrite `regexp` query (#42711) + +commit bfb2e323e9344f7cb0706181f982a41fb7671c40 +Author: Ioannis Kakavas +Date: Wed Jul 24 15:00:50 2019 +0300 + + mute test (#44809) + + see #44808 + +commit 5453188ceff239154754e132f9add291e05361ce +Author: Dimitris Athanasiou +Date: Wed Jul 24 14:24:39 2019 +0300 + + [TEST] Mute SharedClusterSnapshotRestoreIT.testParallelRestoreOperationsFromSingleSnapshot + + This was supposed to be muted in #44675 and its backports but that PR accidentally muted + another test. + + Relates #44671 + +commit 4a3218551c47adfd1d38df3aa6859637edd153f8 +Author: Armin Braun +Date: Wed Jul 24 13:12:57 2019 +0200 + + Fix ConnectionManagerTests (#44769) (#44789) + + * In both fake connection validators we were potentially executing the listener twice. This lead to the situation that the locking via `connectionLock` that ensures that each listener is only executed once ever + would fail and the lister would run twice (in which case the listeners for that node are already `null` and we get an NPE) + * The fact that two different tests fail is due to the fact that we weren't safely shutting down the threadpool which meant the the task that trips the assertion (on the generic pool) would leak into the next test and fail it + * Closes #44758 + +commit c329b454d9487658b4263c1ea245989da48f6a08 +Author: Jason Tedor +Date: Wed Jul 24 17:15:11 2019 +0900 + + Mark fields in SystemdPluginTests as final + + These fields can be final, since they are set at construction, and + changing them after that could lead to some confusing test cases. This + commit allows the compiler to enforce that we never modify these values + during tests. + +commit 26da573e945a0a4607248f590b23f59b60166522 +Author: Przemysław Witek +Date: Wed Jul 24 10:03:25 2019 +0200 + + [ML] [7.x] Only emit deprecation warning if there was actual change of a datafeed's job_id. (#44755) + + * Only emit deprecation warning if there was actual change of a datafeed's job_id. + * Add @Deprecated annotation to DatafeedUpdate.Builder#setJobId method + +commit 58a4bad12ff073181c828fd2903d3436a67c4510 +Author: Jason Tedor +Date: Wed Jul 24 16:30:59 2019 +0900 + + Align assertion and enable check in systemd plugin + + This commit more closely aligns the assertion that we are running in a + package distribution with disabling the systemd integration if somehow + we running on not a package distribution. This is, previously we had an + assertion that we are in a package distribution (RPM or Debian package) + but would disable the systemd integration if we are not on + Linux. Instead, we should disable the systemd integration if we are not + running in a package distribution. Because of our assertion, we expect + this to never hold, but we need a fallback for when this assertion is + violated and assertions are not enabled. + +commit 1e9c505e958d77c11dbedd8e51bd481a4e5fe8d6 +Author: Jason Tedor +Date: Wed Jul 24 00:04:00 2019 -0700 + + Avoid dumping the heap in Painless tests (#44782) + + Well, we have a test here that intentionally causes an OutOfMemoryError, + to ensure that Painless handles it (I still strongly disagree with doing + this). This causes two things to happen: an OutOfMemoryError to be + dumped to the console, and the heap to be dumped to disk. This makes it + look like we had an OutOfMemoryError while running tests, and the tests + did not fail properly. This commit changes the tests configuration so + that we suppress the heap dump, which also causes the OutOfMemoryError + to no longer be dumped to the console. + +commit 4c77d5e2c79f093ce1500bb0c5c99bae99354fa6 +Author: Jason Tedor +Date: Tue Jul 23 23:55:17 2019 -0700 + + Remove stale permissions from untrusted policy (#44783) + + We have some old permissions lying around, granted to untrusted code + from the days of yore when we supported Groovy and Javascript + scripting. This commit removes these stale permissions. + +commit 659ebf6cfb643604fad65b494c4ec0b66adedf4d +Author: Jason Tedor +Date: Tue Jul 23 22:03:48 2019 -0700 + + Notify systemd when Elasticsearch is ready (#44673) + + Today our systemd service defaults to a service type of simple. This + means that systemd assumes Elasticsearch is ready as soon as the + ExecStart (bin/elasticsearch) process is forked off. This means that the + service appears ready long before it actually is, so before it is ready + to receive requests. It also means that services that want to depend on + Elasticsearch being ready to start can not as there is not a reliable + mechanism to determine this. This commit changes the service type to + notify. This requires that Elasticsearch sends a notification message + via libsystemd sd_notify method. This commit does that by using JNA to + invoke this native method. Additionally, we use this integration to also + notify systemd when we are stopping. + +commit 27440b76922a3cf693a84566214ac6b8a58976d3 +Author: Jake Landis +Date: Tue Jul 23 16:28:10 2019 -0500 + + update 7.2.1 release notes (#44774) + +commit c357f81aa79a3df079e87fea022f747737ec0eb3 +Author: James Baiera +Date: Tue Jul 23 15:10:21 2019 -0400 + + Add soft limit for max concurrent policy executions (#43117) + + Adds a global soft limit on the number of concurrently executing enrich policies. + Since an enrich policy is run on the generic thread pool, this is meant to limit + policy runs separately from the generic thread pool capacity. + +commit 818103ff1ea824053d25d2a217bb08503a2b288c +Author: Armin Braun +Date: Tue Jul 23 21:29:07 2019 +0200 + + Fix testRetentionLeasesClearedOnRestore (#44754) (#44766) + + * Fix this test randomly failing when running into async translog persistence edge case and failing to successfully close index + * Also, slightly improve debug logging on close failure + * Closes #44681 + +commit 9338fc853636078005faf63151495c8369796c17 +Author: Igor Motov +Date: Mon Jul 22 08:51:03 2019 -0400 + + GEO: Switch to using GeoTestUtil to generate random geo shapes (#44635) + + Switches to more robust way of generating random test geometries by + reusing lucene's GeoTestUtil. Removes duplicate random geometry + generators by moving them to the test framework. + + Closes #37278 + +commit 9524d71159163db85fe0fda377f4790d33491da6 +Author: James Rodewig +Date: Tue Jul 23 14:04:40 2019 -0400 + + [DOCS] Document // TEARDOWN test snippet. (#44762) + + The `// TEARDOWN` test snippet was added with #34716. You can use this + snippet to end and clean up a test series started with `// TESTSETUP` or + `// TEST[setup:name]`. + +commit caf9411a723f3616edcb0856faed22c307e1f95d +Author: David Roberts +Date: Tue Jul 23 18:00:50 2019 +0100 + + [ML] Improve response format of data frame stats endpoint (#44743) + + This change adjusts the data frame transforms stats + endpoint to return a structure that is easier to + understand. + + This is a breaking change for clients of the data frame + transforms stats endpoint, but the feature is in beta so + stability is not guaranteed. + + Backport of #44350 + +commit 6f53865fdea3819589c2560e951d57a0f1022c7a +Author: Benjamin Trent +Date: Tue Jul 23 08:33:12 2019 -0500 + + [ML][Data Frame] Fixes failure state tests and failure setting handling (#44645) (#44698) + + * [ML][Data Frame] fixing flaky test + + * adjusting frequency + + * fixing tests + + * addressing PR comments + +commit 16c8e18013ef3ff8b84ed7bc2232e3e6ac13c80e +Author: Przemysław Witek +Date: Tue Jul 23 14:48:56 2019 +0200 + + Deprecate the ability to update datafeed's job_id. (#44691) (#44742) + +commit 7f6e2aae82b51628e7cc65761984ce8221416b95 +Author: Ioannis Kakavas +Date: Tue Jul 23 13:42:41 2019 +0300 + + Handle version parsing in RestClientBuilder (#44737) + + Handle version parsing more leniently in RestClientBuilder for the + cases where TLS version pinning is required since we cannot use + JavaVersion here. + +commit 4dd9238cc0d2b5a13d04da884399b64c13dde478 +Author: Ioannis Kakavas +Date: Tue Jul 23 13:16:22 2019 +0300 + + Mute testPooledMemoryChoiceOnNotSmallHeap + +commit e5bd3ad0e9f3969b621a9c049a125ce589ff1fda +Author: Armin Braun +Date: Tue Jul 23 10:52:37 2019 +0200 + + Remove some Dead Code in o.e.transport (#44653) (#44734) + + * None of this is used + +commit ee23968f0509fc8dc7ffb79b79341b2d4b3b199f +Author: David Turner +Date: Tue Jul 23 08:43:09 2019 +0100 + + Ignore unknown fields if overriding node metadata (#44689) + + The `elasticsearch-node override-version` command fails if it cannot read the + existing node metadata file. However, it reads this file strictly and fails if + there are any unknown fields, which means it will not be useful if we add + another field in future. + + This commit adds leniency to this command, allowing it to ignore any unknown + fields and proceed with the downgrade. A downgrade is already unsafe, and the + user is already copiously warned about this, so being lenient in this case does + not make things much worse. + +commit 6928a315c4d35380757f27472df591e261bf0245 +Author: Jason Tedor +Date: Tue Jul 23 00:49:16 2019 -0700 + + Check shard limit after applying index templates (#44619) + + Today when creating an index and checking cluster shard limits, we check + the number of shards before applying index templates. At this point, we + do not know the actual number of shards that will be used to create the + index. In a case when the defaults are used and a template would + override, we could be grossly underestimating the number of shards that + would be created, and thus incorrectly applying the limits. This commit + addresses this by checking the shard limits after applying index + templates. + +commit e2c8f8dfa3b4dbfb4539da3833f90047dd4c57c8 +Author: Jason Tedor +Date: Mon Jul 22 20:32:29 2019 -0700 + + Rename ILM package to ilm (#44725) + + This commit renames the ILM package from indexlifecycle to ilm. We have + all come to know index lifecycle management as ILM, the APIs and + settings use ilm, and it would be nice of the package did too. This + commit makes that change. + +commit 05ec970723534940f6d18a1a568687b543c8f720 +Author: Ignacio Vera +Date: Tue Jul 23 09:05:47 2019 +0200 + + Support BucketScript paths of type string and array. (#44694) (#44731) + +commit 8516fb0f3bb3c49babbfd8fa93fdad3bab9598c1 +Author: David Turner +Date: Tue Jul 23 07:32:44 2019 +0100 + + Expand docs on force-merge and global ordinals (#44684) + + Some small clarifications about force-merging and global ordinals, particularly + that global ordinals are cheap on a single-segment index and how this relates + to frozen indices. + + Fixes #41687 + +commit ed8f75c990ef01de006b4e5f15c1aaa07bfac9dc +Author: Lisa Cawley +Date: Mon Jul 22 15:29:59 2019 -0700 + + [DOCS] Adds more validation details for data frame transforms (#44714) + +commit 5878bde8dc8de6b1745a61b07703e21cecb75c72 +Author: Jason Tedor +Date: Mon Jul 22 15:32:44 2019 -0700 + + Rename SLM package to slm (#44608) + + This commit renames the SLM package from snapshotlifecycle to slm. We + have all come to know index lifecycle management as ILM, the APIs and + settings use ilm, and it would be nice of the package did too. For SLM, + let's use slm for all of these including the package name from the + beginning. + +commit 4456850a8e59c75f403e24dac0e9469030157f13 +Author: Benjamin Trent +Date: Mon Jul 22 15:12:55 2019 -0500 + + [7.x] [ML][Data Frame] Add optional defer_validation param to PUT (#44455) (#44697) + + * [ML][Data Frame] Add optional defer_validation param to PUT (#44455) + + * [ML][Data Frame] Add optional defer_validation param to PUT + + * addressing PR comments + + * reverting bad replace + + * addressing pr comments + + * Update put-transform.asciidoc + + * Update put-transform.asciidoc + + * Update put-transform.asciidoc + + * adjusting for backport + + * fixing imports + + * [DOCS] Fixes formatting in create data frame transform API + +commit fc20264b99a437b309ffb1fb1d12be6652263baa +Author: James Baiera +Date: Wed Jul 17 12:57:41 2019 -0400 + + Add Enrich index background task to cleanup old indices (#43746) + + This PR adds a background maintenance task that is scheduled on the master node only. + The deletion of an index is based on if it is not linked to a policy or if the enrich alias is not + currently pointing at it. Synchronization has been added to make sure that no policy + executions are running at the time of cleanup, and if any executions do occur, the marking + process delays cleanup until next run. + +commit 06e21f79026e8f776f158aec3e19004f14d043ba +Author: Benjamin Trent +Date: Mon Jul 22 13:13:25 2019 -0500 + + [7.x] [ML][Data Frame] adding force delete (#44590) (#44696) + + * [ML][Data Frame] adding force delete (#44590) + + * [ML][Data Frame] adding force delete + + * Update delete-transform.asciidoc + + * adjusting for backport + +commit 3714cb63da29b20df709ecc9606a0ebbc2d2a561 +Author: Ioannis Kakavas +Date: Mon Jul 22 20:13:32 2019 +0300 + + Allow parsing the value of java.version sysprop (#44017) + + We often start testing with early access versions of new Java + versions and this have caused minor issues in our tests + (i.e. #43141) because the version string that the JVM reports + cannot be parsed as it ends with the string -ea. + + This commit changes how we parse and compare Java versions to + allow correct parsing and comparison of the output of java.version + system property that might include an additional alphanumeric + part after the version numbers + (see [JEP 223[(https://openjdk.java.net/jeps/223)). In short it + handles a version number part, like before, but additionally a + PRE part that matches ([a-zA-Z0-9]+). + + It also changes a number of tests that would attempt to parse + java.specification.version in order to get the full version + of Java. java.specification.version only contains the major + version and is thus inappropriate when trying to compare against + a version that might contain a minor, patch or an early access + part. We know parse java.version that can be consistently + parsed. + + Resolves #43141 + +commit 2a4380e0aae7b72daaf2bde0906a43f77653a796 +Author: David Roberts +Date: Mon Jul 22 17:52:39 2019 +0100 + + [DOCS] Clarifies data frame transform frequency is also for retries (#44229) + + In data frame transforms the same scheduler controls both + retries in the event of search failures and gaps between + checks for changes when the transform is running continuously. + + Co-Authored-By: Lisa Cawley + +commit bcb3563dcfff040fbd7ae4b01d0bcfbbc7c038fc +Author: Tanguy Leroux +Date: Mon Jul 22 17:09:25 2019 +0200 + + Remove AllocationService.reroute(ClusterState, String, boolean) (#44629) + + This commit removes the method AllocationService.reroute(ClusterState, String, boolean) + in favor of AllocationService.reroute(ClusterState, String). + + Motivations are: + there are already 3 other reroute methods in this class + this method is always called with the debug parameter set to false + almost all tests use the method reroute(ClusterState, String) + +commit 4387d81e5b22318c1ce2841363dd474f97f0fe19 +Author: Hendrik Muhs +Date: Mon Jul 22 15:42:05 2019 +0200 + + add a test to check excecution flow (#44481) + + add a test for the execution flow of a2p indexer + +commit 5273a548a49d8d29be6e78ce56e333fc61bb2389 +Author: Evgenia Badiyanova +Date: Mon Jul 22 10:59:21 2019 -0400 + + Unmute PendingTasksBlocksIT tests + +commit a948362d0ac84634aef9ec6cacb7fa284451a945 +Author: Benjamin Trent +Date: Mon Jul 22 09:06:48 2019 -0500 + + [7.x] [ML][Data Frame] deregister scheduler on transform failure (#44569) (#44576) + + * [ML][Data Frame] deregister scheduler on transform failure (#44569) + + * fixing test + + * Update DataFrameRestTestCase.java + + * Update DataFrameTaskFailedStateIT.java + + * Update DataFramePivotRestIT.java + +commit 6ceae5d586b9f71480ad2981b1aa237f9bcce9bb +Author: Armin Braun +Date: Mon Jul 22 16:06:34 2019 +0200 + + Document Type of Collections Returned by StreamInput (#44686) (#44688) + + * As a result of #44665 the collections returned by the deserialization methods on `StreamInput` may be either mutable or immutable now, + this PR adds documentation for that fact + +commit 8ee4c4d5ba7938f1a91909444b7bf3b6a8d6a209 +Author: Evgenia Badiyanova +Date: Mon Jul 22 09:53:30 2019 -0400 + + Mute some tests in PendingTasksBlocksIT + + Tracked in #44695. + +commit dcb3b2c18ab221e44bb4e2e85638221ed1d394cc +Author: David Turner +Date: Mon Jul 22 14:39:41 2019 +0100 + + Fix testPendingTasksWithClusterNotRecoveredBlock + + In 7.x we cannot start a new master-eligible node before the cluster has formed + since we first try and update minimum_master_nodes and this is blocked. This + commit changes the test to start a data-only node so that no such adjustment is + necessary. + + Relates #44685 + +commit f5b2fd2f1a8d30420eca3003d80f807d59e56762 +Author: Jason Tedor +Date: Mon Jul 22 22:33:41 2019 +0900 + + Fix imports in JvmErgonomicsTests.java + +commit 5c0ebe7b5f5261f5553e4d61112a4cbf1182bcad +Author: Jason Tedor +Date: Mon Jul 22 22:26:41 2019 +0900 + + Reenable JvmErgonomicsTests on Windows + +commit 8d7392de35da6a3e312f0e9b60ceb168336fa99e +Author: James Rodewig +Date: Mon Jul 22 08:51:34 2019 -0400 + + [DOCS] Make field datatype titles consistent (#43933) + + * [DOCS] Make field datatype titles consistent + + * Add titleabbrev for array + +commit 972a49312ca9c342bef47caded5ac1025872468d +Author: Mayya Sharipova +Date: Fri Jul 19 12:05:55 2019 -0400 + + Fix testQuotedQueryStringWithBoost test (#43385) + + Add more logging to indexRandom + + Seems that asynchronous indexing from indexRandom sometimes indexes + the same document twice, which will mess up the expected score calculations. + + For example, indexing: + { "index" : {"_id" : "1" } } + {"important" :"phrase match", "less_important": "nothing important"} + { "index" : {"_id" : "2" } } + {"important" :"nothing important", "less_important" :"phrase match"} + Produces the expected scores: 13.8 for doc1, and 1.38 for doc2 + + indexing: + { "index" : {"_id" : "1" } } + {"important" :"phrase match", "less_important": "nothing important"} + { "index" : {"_id" : "2" } } + {"important" :"nothing important", "less_important" :"phrase match"} + { "index" : {"_id" : "3" } } + {"important" :"phrase match", "less_important": "nothing important"} + Produces scores: 9.4 for doc1, and 1.96 for doc2 which are found in the + error logs. + + Relates to #43144 + +commit a154f49b944d7c542d61ab53bf30fd31dd4cac4e +Author: Przemyslaw Gomulka +Date: Mon Jul 22 14:28:39 2019 +0200 + + Fix stats in slow logs to be a escaped JSON backport(#44642) #44687 + + Fields in JSON logs should be an escaped JSON fields. It is a broken json value at the moment + "stats": "["group1", "group2"]", -> "stats": "[\"group1\", \"group2\"]", + This should later be refactored into a JSON array of strings (the same as types in 7.x) + +commit 0ce3114779a3cd74953a52c2bdcdbfb515f0e57c +Author: David Turner +Date: Mon Jul 22 13:14:19 2019 +0100 + + Allow pending tasks before state recovery (#44685) + + Today we block access to the pending tasks API before the cluster has recovered + its state. There's no real need to do so, and the master does meaningful work + even before performing state recovery so it might sometimes be useful to allow + access to this API. This commit changes this API to ignore all cluster blocks. + + Fixes #44652 + +commit 6d27eec30f35b7abead14102c88888b893a5f998 +Author: David Roberts +Date: Mon Jul 22 11:28:24 2019 +0100 + + [ML-DataFrame] Use lenient expand open in data frame searches (#44633) + + Since #44344 we use IndicesOptions.LENIENT_EXPAND_OPEN + when deciding which indices to include in checkpoint + calculation. This change uses the same option when + deciding which indices to search for data and which + indices to get mappings from, otherwise there is a + potential mismatch between the checkpoint details and + what is searched elsewhere. + +commit 09e9c4cb593ff217a197c8e6a32eb33a39ea295f +Author: Przemyslaw Gomulka +Date: Mon Jul 22 12:02:20 2019 +0200 + + Fix types field in JSON Search Slow Logs (#44641) + + The field has to be defined in log4j2.properties and should be an + escaped JSON for now (it is a broken JSON at the moment). This should later be refactored into a JSON array + of strings. + +commit fe20e217a46909994c85f141487da688861059da +Author: Przemyslaw Gomulka +Date: Mon Jul 22 11:38:11 2019 +0200 + + Deprecation messages with the same key but different x-opaque-id are allowed backport(#44587) #44682 + + Deprecation logger was filtering log entries by key, that means that if two log messages with the same key are logged from different users, then the second log messages will be filtered. + This change allows to log deprecation message with the same key by different users. + + relates #41354 + backport #44587 + +commit a6adcecd20290f8c0092af528e0d8d268660f188 +Author: Armin Braun +Date: Mon Jul 22 10:47:28 2019 +0200 + + Fix Tring to Mutate Immutable Collections + + Fixes two spots where #44665 caused a previously mutable collection to now be read as an immutable one, leading to errors + +commit b9067ba1ba6eb81fa17024dde7b90eaba18f5159 +Author: Armin Braun +Date: Mon Jul 22 10:57:30 2019 +0200 + + Remove Needless Synchronization in FollowersChecker (#44631) (#44680) + + * It seems redundant to synchronize here and check that the map hasn't checked via the `isRunning` under the mutex + * The map won't change if under the mutex that locks on all the updates to it + * Without the mutex it's very unlikely to change inside the method call relative to the likelihood of changing until the generic pool where we check for `isRunning` again anyway + + -> just remove the synchronization (it's on the IO loop) and check since we do check the running state on the generic pool under the mutex anyway when we actually fail it + +commit ff76b0af8bbfe8e67476b1bc82142b2c5f1c0962 +Author: Jason Tedor +Date: Mon Jul 22 17:20:21 2019 +0900 + + Copy field names in stored fields context + + We have to copy the field names otherwise we either have a handle of a + list that a caller might mutate or we might mutate when they aren't + expecting it, or worse, a handle of a list that is not mutable (and we + end up mutating the list). + + Relates #44665 + +commit b34ac66d96cae456529e5e04c1852593551c3e13 +Author: Alpar Torok +Date: Mon Jul 22 11:32:29 2019 +0300 + + Mute multiple tests on Windows (7.x) (#44676) + + * Mute failing test + + tracked in #44552 + + * mute EvilSecurityTests + + tracking in #44558 + + * Fix line endings in ESJsonLayoutTests + + * Mute failing ForecastIT test on windows + + Tracking in #44609 + + * mute BasicRenormalizationIT.testDefaultRenormalization + + tracked in #44613 + + * fix mute testDefaultRenormalization + + * Increase busyWait timeout windows is slow + + * Mute failure unconfigured node name + + * mute x-pack internal cluster test windows + + tracking #44610 + + * Mute JvmErgonomicsTests on windows + + Tracking #44669 + + * mute SharedClusterSnapshotRestoreIT testParallelRestoreOperationsFromSingleSnapshot + + Tracking #44671 + + * Mute NodeTests on Windows + + Tracking #44256 + +commit d82a570a2acfa3457a4942e1ce84669b51caf7fe +Author: Jason Tedor +Date: Mon Jul 22 16:54:16 2019 +0900 + + Reomve debugging loging statements from Azure tests + + This commit removes some unneeded debugging logging statements from the + Azure storage tests. + + Relates #44672 + +commit a493a341436075ddf7329609295ad8ded6327a10 +Author: Jason Tedor +Date: Mon Jul 22 00:44:46 2019 -0700 + + Use debug logging instead for Azure tests (#44672) + + These Azure tests have hard println statements which means we always see + these messages during configuration. Yet, there are unnecessary most of + the time. This commit changes them to use debug logging. + +commit 0e2e83f591045a544f0e3e2a39c9f7637d1de9ac +Author: Armin Braun +Date: Mon Jul 22 09:31:12 2019 +0200 + + More Efficient Deserialization of Empty Collections in StreamInput (#44665) (#44674) + + * We only had the `size == 0` optimization in some but not all spots of deserializing collections in this class, fixed the remaining spots. + * Also fixed the a similar spot when deserializing `ThreadContextStruct` that could now be simplified (it was apparently doing it's own version of this optimization for the first map it deserialized before ... but not for the second map -> made it not instantiate anything if both maps are empty since it's always the same object here anyway) + +commit 0ac137a9a11766c41cedb0832f3f336342358c13 +Author: Armin Braun +Date: Mon Jul 22 07:09:32 2019 +0200 + + Optimize some StreamOutput Operations (#44660) (#44668) + + * Optimize some StreamOutput Operations + + * Writing numbers byte by byte adds a lot of unnecessary bounds checks to serialization + * Serializing to a threadlocal `byte[]` instead and bulk writing gives about a 50% speedup on `long` and `vlong` (for large numbers) writes and 30% for `int`, `vint` on Linux on an i9 + * Using a threadlocal of the maximum string buffer size we used to allocate before also removes allocations when writing strings in general since we now never have to allocate a `byte[]` for that + * And don't have to GC one either resolving the TODO removed here + +commit 226a753e937fe37f22b40fb03e98a287bae0ada9 +Author: Ryan Ernst +Date: Sun Jul 21 13:54:46 2019 -0700 + + Restore setting up temp dir for windows service (#44541) (#44661) + + In https://github.com/elastic/elasticsearch/pull/41913 setting up the + temp dir for ES was moved from the env script to individual cli scripts. + However, moving it to the windows service cli was missed. This commit + restores setting up the temp dir for the windows service control script. + +commit 1a9cfe9110445179367662e90951d82436c7ad5b +Author: Tal Levy +Date: Sat Jul 20 19:10:49 2019 -0700 + + Removal Streamable (#44647) (#44655) + + This commit ends the grand adventure that was the + refactoring effort to migrate all usages of + Streamable to Writeable. + + Closes #34389. + +commit cdd06d40d2ef012521184b5ddcc6e3dad2cf3eba +Author: Jason Tedor +Date: Sat Jul 20 15:18:16 2019 -0700 + + Do not checksum all bytes at once in plugin install (#44649) + + Today when checksumming a plugin zip during plugin install, we read all + of the bytes of the zip into memory at once. When trying to run the + plugin installer on a small heap (say, 64 MiB), this can lead to the + plugin installer running out of memory when checksumming large + plugins. This commit addresses this by reading the plugin bytes in 8 KiB + chunks, thus using a constant amount of memory independent of the size + of the plugin. + +commit 4c05d25ec7ddd4049bba5eed9b3cb51f50f0fee7 +Author: Ryan Ernst +Date: Sat Jul 20 11:25:58 2019 -0700 + + Convert Transport Request/Response to Writeable (#44636) (#44654) + + This commit converts all remaining TransportRequest and + TransportResponse classes to implement Writeable, and disallows + Streamable implementations. + + relates #34389 + +commit f4ee2e9e91e324433614934e5b672060efd75614 +Author: Ryan Ernst +Date: Sat Jul 20 08:32:29 2019 -0700 + + Convert direct implementations of Streamable to Writeable (#44605) (#44646) + + This commit converts Streamable to Writeable for direct implementations. + + relates #34389 + +commit 1f7fc1b497736fdb3a49ad420c63700f9c3fd451 +Author: Jason Tedor +Date: Fri Jul 19 17:28:23 2019 -0700 + + Add default CLI JVM options (#44545) + + This commit adds some default CLI JVM options to control the heap size + and the garbage collector used for the CLI tools. We do this because + otherwise the JVM will default to large initial and max heap sizes based + on the RAM visible to the JVM (which could be all the physical RAM on + the machine if not run in a container-aware JVM). This commit therefore + sets the initial heap size to 4m, the max heap size to 64m, the garbage + collector to the serial collector, and leaves this user-configurable by + honoring ES_JAVA_OPTS last. + +commit 7c84636029b5bdcdd760c3fb62b181bb2e1ca8a8 +Author: Tal Levy +Date: Fri Jul 19 15:55:53 2019 -0700 + + Remove StreamOutput #writeOptionalStreamable and #writeStreamableList (#44602) (#44643) + + remove usages of writeOptionalStreamable and writeStreambaleList + + relates #34389. + +commit 2e303fc5f7c52a4cd0ced3585929bbe2c02007a5 +Author: Benjamin Trent +Date: Fri Jul 19 16:17:39 2019 -0500 + + [ML][Data Frame] adding dynamic cluster setting for failure retries (#44577) (#44639) + + This adds a new dynamic cluster setting `xpack.data_frame.num_transform_failure_retries`. + + This setting indicates how many times non-critical failures should be retried before a data frame transform is marked as failed and should stop executing. At the time of this commit; Min: 0, Max: 100, Default: 10 + +commit f193d147642627fd94278a2940fd77c716d79fe8 +Author: Ryan Ernst +Date: Fri Jul 19 13:33:38 2019 -0700 + + Convert remaining Action Response/Request to writeable.reader (#44528) (#44607) + + This commit converts readFrom to ctor with StreamInput on the remaining + ActionResponse and ActionRequest classes. + + relates #34389 + +commit a63f60b77603931b98733cf44689c1f95869d032 +Author: James Rodewig +Date: Fri Jul 19 14:35:36 2019 -0400 + + [DOCS] Remove heading offsets for REST APIs (#44568) + + Several files in the REST APIs nav section are included using + :leveloffset: tags. This increments headings (h2 -> h3, h3 -> h4, etc.) + in those files and removes the :leveloffset: tags. + + Other supporting changes: + * Alphabetizes top-level REST API nav items. + * Change 'indices APIs' heading to 'index APIs.' + * Changes 'Snapshot lifecycle management' heading to sentence case. + +commit f028ab43ad23cdc3e2de942864e8ad72952ef215 +Author: Armin Braun +Date: Fri Jul 19 20:35:29 2019 +0200 + + Don't Swallow Interrupt in TransportService#onRequestReceived (#44622) (#44627) + + * We shouldn't just swallow the interrupt here quietly and keep going on the IO thread + * Currently interrupt continues here just the same way an invocation of `acceptIncomingRequests` woudl have made things continue + * Relates #44610 + +commit eafe54c81c47e80fbcf870306b878134c4c4cec1 +Author: Christoph Büscher +Date: Fri Jul 19 18:15:11 2019 +0200 + + Fix AnalysisMode propagation in NamedAnalyzer (#44626) + + NamedAnalyzer should return the same AnalysisMode than any custom analyzer it + wraps, otherwise AnalysisMode.ALL. This used to be only CustomAnalyzer in the + past, but with the introduction of the ReloadableCustomAnalyzer this needs to be + added as an option where the analysis mode gets propagated. + + Closes #44625 + +commit c3b6ab4dec97fd404b33393e34c20579c26485e2 +Author: David Turner +Date: Fri Jul 19 17:03:05 2019 +0100 + + Fix docs line length + + Followup to #44617 that fixes some unexpected wrapping. + +commit 87f246f6bd96486fff44cb3b94dc5ef0c1e47659 +Author: David Turner +Date: Fri Jul 19 16:07:54 2019 +0100 + + Clarify repository-s3 defaulting to instance role (#44617) + + Specifies more clearly that you can leave the client credential settings unset + if you want to use the instance or container role. + +commit 1f5a2ac3adaf585aae102dc98794672d7c0bbec3 +Author: James Rodewig +Date: Fri Jul 19 11:03:06 2019 -0400 + + [DOCS] Reformat `parent_id` query docs (#44449) + +commit 804476c35d6d3bcdd71193eca424274cbf89bd4c +Author: Nikita Glashenko +Date: Fri Jul 19 20:01:47 2019 +0500 + + Remove support for old translog checkpoint formats (#44280) + + This commit removes support for the translog checkpoint format from versions + before 6.0.0 since 7.x versions are incompatible with indices from these + versions. + + Relates #44720 + Fixes #44210 + +commit 51aefbdd3d0c30b479cd1ce8dc8e5e5166c378a9 +Author: James Rodewig +Date: Fri Jul 19 10:50:36 2019 -0400 + + [DOCS] Reformat `has_parent` query docs (#44443) + +commit d46545f7299082ca905fb321536bcaa017046ba2 +Author: James Rodewig +Date: Fri Jul 19 09:16:35 2019 -0400 + + [DOCS] Update anchors and links for Elasticsearch API relocation (#44500) + +commit 76c7e3a05f6d2e4a56dd5b01ef3dae76be87b06f +Author: James Rodewig +Date: Fri Jul 19 08:40:34 2019 -0400 + + [DOCS] Replace `_meta` with `metadata` for snapshot APIs. (#44596) + + elastic/elasticsearch#41281 added custom metadata parameter to + snapshots. During review, the parameter name was changed from '_meta' to + 'metadata,' but the documentation wasn't updated. This corrects the + documentation to use the 'metadata' name. + +commit 597d2dfaf51c901320643ac28c62f9d361d86ec7 +Author: Przemyslaw Gomulka +Date: Fri Jul 19 08:31:00 2019 +0200 + + Add types field to slow logs in 7.x (#44592) + + By mistake in 7.x types field was removed from slow logs. Types are + still present in that version, so this have to be present as a JSON + field + relates #41354 + backport that was causing this #44178 + +commit 60785a9fa849e57ef85881383dd8fbf4f0de2e16 +Author: Ryan Ernst +Date: Thu Jul 18 21:25:44 2019 -0700 + + Convert several direct uses of Streamable to Writeable (#44586) (#44604) + + This commit converts several utility classes that implement Streamable + to have StreamInput constructors. It also adds a default version of + readFrom to Streamable so that overriding to throw UOE is not necessary. + + relates #34389 + +commit 336364fefe5f9b4306a7ddd64ab9c05347c3e030 +Author: Julie Tibshirani +Date: Thu Jul 18 18:45:10 2019 -0700 + + Convert more classes in 'server' to Writeable. (#44600) + + * Convert GetTask*. + * Convert RemoteInfo*. + * Convert GetFieldMappings*. + * Convert ValidateQueryRequest*. + * Convert MainResponse*. + * Convert MultiGet*. + * Convert Update*. + * Add a missing call to parent constructors. + + Relates to #34389. + +commit 13f46aa801c175d395d6fe47abe616ffb968dd15 +Author: Ryan Ernst +Date: Thu Jul 18 18:32:09 2019 -0700 + + Convert index and persistent actions/response to writeable (#44582) (#44601) + + This commit converts several more classes from streamable to writeable + in server, mostly within the o.e.index and o.e.persistent packages. + + relates #34389 + +commit 03f5084ac7a48379c705353b1db6dae55774cc5c +Author: Tal Levy +Date: Thu Jul 18 16:19:02 2019 -0700 + + remove usages of #readOptionalStreamable, #readStreamableList. (#44578) (#44598) + + This commit removes references to Streamable from StreamInput. + + This is all a part of the effort to remove Streamable usage. + + relates #34389. + +commit 5a05bd76b467dca7889689ec84a58b85d35df3f2 +Author: Lisa Cawley +Date: Thu Jul 18 16:10:22 2019 -0700 + + [DOCS] Reformat info and deprecation APIs to use new API format (#44599) + +commit 3001f7941f3854c1e45b82d9782d11b1990f1053 +Author: Lee Hinman +Date: Thu Jul 18 16:19:49 2019 -0600 + + Allow empty configuration for SLM policies (#44465) + + * Allow empty configuration for SLM policies + + When putting or updating a snapshot lifecycle policy it was not possible + to elide the `config` map. This commit makes the configuration optional, + the same way that it is when taking a snapshot. + + Relates to #38461 + + * Add Objects.requireNonNull for required parts of the policy + +commit fe2ef66e45f1691b5a488286f41cddc7e26c403b +Author: Lee Hinman +Date: Thu Jul 18 15:32:52 2019 -0600 + + Expose index age in ILM explain output (#44457) + + * Expose index age in ILM explain output + + This adds the index's age to the ILM explain output, for example: + + ``` + { + "indices" : { + "ilm-000001" : { + "index" : "ilm-000001", + "managed" : true, + "policy" : "full-lifecycle", + "lifecycle_date" : "2019-07-16T19:48:22.294Z", + "lifecycle_date_millis" : 1563306502294, + "age" : "1.34m", + "phase" : "hot", + "phase_time" : "2019-07-16T19:48:22.487Z", + ... etc ... + } + } + } + ``` + + This age can be used to tell when ILM will transition the index to the + next phase, based on that phase's `min_age`. + + Resolves #38988 + + * Expose age in getters and in HLRC + +commit af093a409545dd69783b73c2743246c1b4e69817 +Author: Ryan Ernst +Date: Thu Jul 18 13:29:19 2019 -0700 + + Convert ShardOperationFailedException to Writeable (#44532) (#44580) + + This commit converts subclasses of ShardOperationFailedException to + implement ctors with StreamInput instead of readFrom. It also simplifies + IndicesShardStoresResponse.Failure to serialize its shardId after the + super data. + + relates #34389 + +commit 3477f5ae04d3efe1705a5d34b487b8dee5a3e473 +Author: Benjamin Trent +Date: Thu Jul 18 15:03:50 2019 -0500 + + muting test testBulkIndexFailuresCauseTaskToFail (#44594) + +commit d5ca72740e3f548d6832e9e200e27d436a6dbe2e +Author: Benjamin Trent +Date: Thu Jul 18 14:28:34 2019 -0500 + + [ML][Data Frame] adjust onFinish audit frequency (#44450) (#44508) + +commit 8745d38dc8040d0a2cf7c2ed5193ef2b764c354e +Author: Marios Trivyzas +Date: Thu Jul 18 21:15:56 2019 +0300 + + Compile subset of build-tools classes with Java 8 target compatibility (#44560) + + Backport of #43177 so that VersionProperties is Java 8 compatible and + can be used by https://github.com/elastic/elasticsearch-hadoop + to retrieve snapshot versions for Lucene. + + (cherry picked from commit ec3ac9b62452f04ce44dea0a904a6e2b31dd8076) + +commit 3b5038b837927fb038b4a1cf0388f98a25e2e9fe +Author: Armin Braun +Date: Thu Jul 18 17:54:54 2019 +0200 + + Implement Eventually Consistent Mock Repository for SnapshotResiliencyTests (#40893) (#44570) + + * Add eventually consistent mock repository for reproducing and testing AWS S3 blob store behavior + * Relates #38941 + +commit 8445c41004ceaf1d3cdc7d56d64a9a1765ae01ae +Author: Lisa Cawley +Date: Thu Jul 18 08:44:52 2019 -0700 + + [DOCS] Moves content to ML anomaly-detection folder (#44520) (#44530) + +commit c8a8915b277cb2e94b092503dde8abb14335a6dd +Author: Tal Levy +Date: Thu Jul 18 08:42:56 2019 -0700 + + migrate rollup/monitoring/graph/watcher actions to Writeable (#44464) (#44538) + + this commit migrates leftover actions from a few x-pack plugins + to the new Writeable.Reader infrastructure. + + relates #34389. + +commit 651b7a001f17c94a2b220aeef80a59f73c584c61 +Author: Dimitrios Liappis +Date: Thu Jul 18 18:39:55 2019 +0300 + + Support log file rolling in jvm presence packaging tests (#44548) + + Make JVM presence related packaging tests more resilient, by + also checking for entries in rolled over log files. + + Closes #41436 + Backport of #44423 + +commit ef6ddd15c62fb6e459d5bf9fc680d98898d70f9d +Author: Andrey Ershov +Date: Thu Jul 18 17:19:49 2019 +0200 + + Revert "Snapshot tool: S3 orphaned files + cleanup (#44551)" + + This reverts commit 09edeeb3 + +commit 213af8411f01cbc971068058a18cfd00bb464469 +Author: Lisa Cawley +Date: Thu Jul 18 08:15:28 2019 -0700 + + [DOCS] Fixes query default value (#44572) + +commit 47cfc25fa05682581ee9cd92377cccffb1afac9e +Author: Nhat Nguyen +Date: Thu Jul 18 09:37:25 2019 -0400 + + Skip update if leader and follower settings identical (#44535) + + If the setting on the follower and the leader are identical after + filtering out private and internal settings, then we should not call + update setting (on the follower) as there's nothing to change. Moreover, + this makes the ShardFollowTask abort as it considers + ActionRequestValidationException (caused by an empty update setting + request) as a fatal error. + + Closes #44521 + +commit fe23dd1073ae0d54f63779fc544bbef12d3f85fe +Author: Lisa Cawley +Date: Thu Jul 18 07:58:28 2019 -0700 + + [DOCS] Clarifies data transform source definition (#44504) + +commit 6f5327ba45171d4b48e4af73281203c7c19ffead +Author: Andrey Ershov +Date: Thu Jul 18 17:00:23 2019 +0200 + + Fix BlobStoreTestUtil + +commit 07cf2cb665d5048411e38916c40c9a20273559e5 +Author: Armin Braun +Date: Thu Jul 18 16:57:56 2019 +0200 + + Add disable_chunked_encoding Setting to S3 Repo (#44052) (#44562) + + * Add disable_chunked_encoding setting to S3 repo plugin to support S3 implementations that don't support chunked encoding + +commit 94ffa5a93cf17db8b57f2842fae221e810bcb197 +Author: Lisa Cawley +Date: Thu Jul 18 07:43:43 2019 -0700 + + [DOCS] Improves description of dest data frame transform object (#44509) + +commit 09edeeb38e95dbd159adf70fe2575cc318a72b20 +Author: Andrey Ershov +Date: Thu Jul 18 17:35:10 2019 +0200 + + Snapshot tool: S3 orphaned files cleanup (#44551) + + A tool to work with snapshots. + Co-authored by @original-brownbear. + This commit adds snapshot tool and the single command cleanup, that + cleans up orphaned files for S3. + Snapshot tool lives in x-pack/snapshot-tool. + + (cherry picked from commit fc4aed44dd975d83229561090f957a95cc76b287) + +commit 661f70183e3a607dbb2a61b0b85a1df479f78435 +Author: James Rodewig +Date: Thu Jul 18 10:18:11 2019 -0400 + + [DOCS] Make Query DSL titles consistent (#43935) + +commit 452f7f67a015f407585c06ff9adbc0d80531a423 +Author: David Turner +Date: Thu Jul 18 14:10:40 2019 +0100 + + Defer reroute when starting shards (#44539) + + Today we reroute the cluster as part of the process of starting a shard, which + runs at `URGENT` priority. In large clusters, rerouting may take some time to + complete, and this means that a mere trickle of shard-started events can cause + starvation for other, lower-priority, tasks that are pending on the master. + + However, it isn't really necessary to perform a reroute when starting a shard, + as long as one occurs eventually. This commit removes the inline reroute from + the process of starting a shard and replaces it with a deferred one that runs + at `NORMAL` priority, avoiding starvation of higher-priority tasks. + + Backport of #44433 and #44543. + +commit 4c95cc3223245796947b2e07d0f12389eaee6754 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Jul 18 15:07:33 2019 +1000 + + skip repository-hdfs integTest in case of fips jvm (#44319) + + The repository-hdfs runners need to be disabled it in fips mode. + + Testing done for all the tasks, dynamic created and static (integTest, integTestHa, integSecureTest, integSecureHaTest) + +commit ec0a0a41db1f8118800279e0714854c15273415a +Author: Alan Woodward +Date: Thu Jul 18 10:26:48 2019 +0100 + + Remove type parameter from ParserContext (#44478) + + ParserContext.getType() is never called, so we can remove it and tidy up + the callers as well. + +commit a8a16e6b08e7be9dd2a2c801b396067b5623c500 +Author: Luca Cavanna +Date: Thu Jul 18 11:04:05 2019 +0200 + + Associate sub-requests to their parent task in multi search API (#44492) + + Multi search accepts multiple search requests and runs them as + independent requests, each one as part of their own search task. Today + they don't get associated though with their parent multi search task, + which would be useful to monitor which msearch a certain search was part + of, if any, and also to cancel all of the sub-requests in case the + parent msearch gets cancelled (though this will also require making + the multi search task cancellable as a follow-up). + +commit e9f257b4d9ec2c54c12b0e5e925426873c01e240 +Author: Christoph Büscher +Date: Thu Jul 18 11:55:22 2019 +0200 + + Fix ReloadDetailsTests compilation + +commit eed2db894720cf101823e6193ab4357ab011b4c9 +Author: Christoph Büscher +Date: Thu Jul 18 11:11:26 2019 +0200 + + Add stream serialization to ReloadAnalyzersResponse (#44420) + + This change adds Writeable support to ReloadAnalyzersResponse that + is required when using the transport client in 7.x. + + Closes #44383 + +commit 1375cc93a86a5d2e0453d59638b75e168edd13d2 +Author: maarab7 <44259533+maarab7@users.noreply.github.com> +Date: Thu Jul 18 12:51:27 2019 +0400 + + Fix parameter value for calling data.advanceExact (#44205) + + While the code works perfectly well for a single segment, it returns the wrong values for multiple segments. E.g. If we have 500 docs in one segment and if we want to get the doc id = 280 then data.advanceExact(topDocs.scoreDocs[i].doc) works fine. If we have two segments, say, with first segment having docs 1-200 and the second segment having docs 201-500, then 280 is fetched from the second segment but is actually 480. Subtracting the docBase (280-200) takes us to the correct document which is 80 in the second segment and actually 280. + +commit 7598e0186a09ab2a73ebc7e3a89f1d0c0bf23ca6 +Author: David Turner +Date: Thu Jul 18 09:50:53 2019 +0100 + + Harmonise indentation of cluster settings (#44540) + + Today the long list of `BUILT_IN_CLUSTER_SETTINGS` is indented differently + between `master` and `7.x`. This sometimes makes backporting painful. This + commit adjusts the indentation of earlier branches to match that in `master`. + +commit 0fc091f166e218660f8e888e50507d807eb366ed +Author: David Kyle +Date: Thu Jul 18 09:33:37 2019 +0100 + + Enable XLint warnings for ML (#44346) + + Removes the warning suppression -Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked. + Many warnings were unchecked warnings in the test code often because of the use of mocks. + These are suppressed with @SuppressWarning + +commit edd26339c56fa40aef3e8b1d95f32f637cf7d11e +Author: Ryan Ernst +Date: Thu Jul 18 01:11:45 2019 -0700 + + Convert remaining request classes in xpack core to writeable.reader (#44524) (#44534) + + This commit converts all remaining classes extending ActionRequest + in xpack core to have a StreamInput constructor. + + relates #34389 + +commit 9f97319b3c7e31ceadd85c0f6b7ac1d342434d16 +Author: Alpar Torok +Date: Thu Jul 18 10:00:19 2019 +0300 + + Detect process third party audit being killed by OOM (#44269) + + * Detect process third party audit being killed by OOM + + It's very common for the third party audit to be killed by the OOM + killer when the system is running low on memory. + Since the forbidden APIs call is expected to fail, we were ignoring + these and incorrectly interpreting the partial output. + With this change we detect and provide a proper error message when this + happens. + +commit 6565825a138882e1abd49a22abaf2bf7837b0a23 +Author: Armin Braun +Date: Thu Jul 18 08:52:37 2019 +0200 + + Avoid CharsRef Allocations in StreamInput (#44488) (#44519) + + * Many messages deserialized from a `StreamInput` only contain short strings, some use-cases of instantiating a `StreamInput` don't deserialize any strings + * Don't allocate `CharsRef` for small strings to save some allocations (especially on the IO threads) + * Lazily allocate a larger `CharsRef` if needed for larger strings like we did before and have it live as long as the `StreamInput` like before as well + +commit 38d2ada84fd3afb75a62a862ec670822ad4fb1d1 +Author: Tal Levy +Date: Wed Jul 17 22:47:09 2019 -0700 + + deprecate Supplier constructors in HandledTransportAction (#44456) (#44533) + + This commit deprecates all constructors of HandledTransportAction + that take in a Supplier instead of a Writeable.Reader for response + objects. + + in addition to the deprecation, the following modules were updated to + leverage Writeable + + - modules:ingest-common + - modules:lang-mustache + + relates #34389. + +commit a5ad59451c19fa928c329ccc5affcfd2e7e19864 +Author: Tal Levy +Date: Wed Jul 17 20:28:29 2019 -0700 + + migrate more ML actions off of using Request suppliers (#44462) (#44529) + + many classes still use the Streamable constructors of HandledTransportAction, + this commit moves more of those classes to the new Writeable constructors. + + relates #34389. + +commit 075a3f0e99b7b277c45c2bb33849d53dccd4a1b0 +Author: Tal Levy +Date: Wed Jul 17 20:28:11 2019 -0700 + + remove usage of ActionType#(String) (#44459) (#44526) + + this commit removes usage of the deprecated + constructor with a single argument and no Writeable.Reader. + + The purpose of this is to reduce the boilerplate necessary for + properly implementing a new action, as well as reducing the + chances of using the incorrect super constructor while classes + are being migrated to Writeable + + relates #34389. + +commit 51180af91dc26a92ba9b41df88565eee88b8ba35 +Author: Nhat Nguyen +Date: Wed Jul 17 08:17:35 2019 -0400 + + Make peer recovery send file chunks async (#44468) + + Relates #44040 + Relates #36195 + +commit 458f24c46a8f2d48450389a0e851fb9cb4e4c6e8 +Author: Nhat Nguyen +Date: Wed Jul 17 13:45:16 2019 -0400 + + Reenable accounting circuit breaker (#44495) + + We have a new Lucene 8.2 snapshot on master and 7.x; hence we can + re-enable the accounting on these branches. + + Relates #30290 + +commit 34c60670188577239b188c054508b52f3cd76463 +Author: Julie Tibshirani +Date: Wed Jul 17 19:04:21 2019 -0700 + + Convert several classes in 'server' to Writeable. (#44527) + + * Convert FieldCapabilities*. + * Convert MultiTermVectors*. + * Convert SyncedFlush*. + * Convert SearchTemplateRequest. + * Convert MultiSearchTemplateRequest. + * Convert GrokProcessorGet*. + * Remove a stray reference to SearchTemplateRequest#readFrom. + + Relates to #34389. + +commit 2a2686e6e7592d70a83298f4f3936367c1542a32 +Author: Ryan Ernst +Date: Wed Jul 17 18:01:45 2019 -0700 + + Convert remaining ActionTypes to writeable in xpack core (#44467) (#44525) + + This commit converts all remaining ActionType response classes to + writeable in xpack core. It also converts a few from server which were + used by xpack core. + + relates #34389 + +commit 17c4b2b839ef40dc269a015818b82ed76c44f7be +Author: Ryan Ernst +Date: Wed Jul 17 18:01:29 2019 -0700 + + Convert MasterNodeRequest to implement Writeable.Reader (#44452) (#44513) + + This commit converts all MasterNodeRequest subclasses to fullfill + Writeable.Reader constructors. + + relates #34389 + +commit 7114fe786ba0b566cad149598e4bc9864157c39d +Author: Paul Sanwald +Date: Wed Jul 17 19:14:16 2019 -0400 + + Fix incorrect calculation of how many buckets will result from a merge operation. (#44461) (#44515) + +commit 8841779de8db641298549d421ab82180bb94e125 +Author: Julie Tibshirani +Date: Wed Jul 17 15:49:38 2019 -0700 + + Convert ClearScroll* to Writeable. (#44511) + + This PR converts `ClearScrollRequest` and `ClearScrollResponse` to + `Writeable`. + + Relates to #34389. + +commit 39c5f98de71708d756d36bfb303a87030e781bf2 +Author: Jason Tedor +Date: Thu Jul 18 05:26:01 2019 +0900 + + Introduce test issue logging (#44477) + + Today we have an annotation for controlling logging levels in + tests. This annotation serves two purposes, one is to control the + logging level used in tests, when such control is needed to impact and + assert the behavior of loggers in tests. The other use is when a test is + failing and additional logging is needed. This commit separates these + two concerns into separate annotations. + + The primary motivation for this is that we have a history of leaving + behind the annotation for the purpose of investigating test failures + long after the test failure is resolved. The accumulation of these stale + logging annotations has led to excessive disk consumption. Having + recently cleaned this up, we would like to avoid falling into this state + again. To do this, we are adding a link to the test failure under + investigation to the annotation when used for the purpose of + investigating test failures. We will add tooling to inspect these + annotations, in the same way that we have tooling on awaits fix + annotations. This will enable us to report on the use of these + annotations, and report when stale uses of the annotation exist. + +commit 621ec7cb84323ef01575f0c92ff1d579e0980936 +Author: Lisa Cawley +Date: Wed Jul 17 13:22:09 2019 -0700 + + [DOCS] Removes out-dated info from Watcher limitations (#44252) + +commit 0755a13c9f3165030e0cc70ada6ecccd93d59a99 +Author: Ryan Ernst +Date: Wed Jul 17 11:17:36 2019 -0700 + + Convert AcknowledgedRequest to Writeable.Reader (#44412) (#44454) + + This commit adds constructors to AcknolwedgedRequest subclasses to + implement Writeable.Reader, and ensures all future subclasses implement + the same. + + relates #34389 + +commit 65fcaecce188930682bcc8486e1afc0605c8c305 +Author: Armin Braun +Date: Wed Jul 17 19:59:00 2019 +0200 + + Remove Minio Host Hack in S3 Repository Build (#44491) (#44497) + + * Resolving the todo to clean this hackyness up + +commit c8b66c549da1511caca3ad2f56da91c8cd9dda57 +Author: Yannick Welsch +Date: Wed Jul 17 17:33:40 2019 +0200 + + Ignore failures to set socket options on Mac (#44355) + + Brings some temporary relief for test failures until #41071 is addressed. + +commit f78e64e3e28e8beaa41138d8c48716a8cafb5356 +Author: Yannick Welsch +Date: Wed Jul 17 17:32:07 2019 +0200 + + Terminate linearizability check early on large histories (#44444) + + Large histories can be problematic and have the linearizability checker occasionally run OOM. As it's + very difficult to bound the size of the histories just right, this PR will let it instead run for 10 seconds + on large histories and then abort. + + Closes #44429 + +commit a19c7977acb1368abaabae67daeefa8143556f9b +Author: Igor Motov +Date: Wed Jul 17 12:23:34 2019 -0400 + + Docs: fix WKTToSQL function example (#44377) + + Fixes wrong example snippet in WKTToSQL documentation. + + Closes #44299 + +commit d3cb7bbc8fa4b08d408451965ad45925a7851c5d +Author: Igor Motov +Date: Wed Jul 17 08:59:06 2019 -0400 + + Geo: fix GeoWKTShapeParserTests (#44448) + + Changes in #44187 introduced some optimization in the way shapes are + generated. These changes were not captured in GeoWKTShapeParserTests. + + Relates #44187 + +commit cd5a3348642ee68ef38866b7eb092f73e2d8f2e3 +Author: Igor Motov +Date: Tue Jul 16 09:37:04 2019 -0400 + + Geo: extract dateline handling logic from ShapeBuilders (#44187) + + Extracts dateline decomposition logic from ShapeBuilder into a separate + utility class that is used on the indexing side. The search side + will be handled as part of another PR at this time we will remove + the decomposition logic from ShapeBuilders as well. This PR also doesn't + change any existing logic including bugs. + + Relates to #40908 + +commit 51d8e6e3de00b6157ccb59f2de711f56b6a7dc22 +Author: Lisa Cawley +Date: Wed Jul 17 08:55:06 2019 -0700 + + [DOCS] Adds sync to data frame transform API (#44254) + +commit b6a0f098e697e9806b24ad4eee52bbe283a3ac73 +Author: Alan Woodward +Date: Wed Jul 17 16:08:28 2019 +0100 + + Don't use index_phrases on graph queries (#44340) + + Due to https://issues.apache.org/jira/browse/LUCENE-8916, when you + try to use a synonym filter with the index_phrases option on a text field, + you can end up with null values in a Phrase query, leading to weird + exceptions further down the querying chain. As a workaround, this commit + disables the index_phrases optimization for queries that produce token + graphs. + + Fixes #43976 + +commit ddd740162e0c0066f5ab458e2310f56e5390cb69 +Author: Yannick Welsch +Date: Wed Jul 17 17:32:47 2019 +0200 + + Do not use CancellableThreads for Zen1 (#44430) + + Zen 1 stops pinging threads in ZenDiscovery by calling Thread.interrupt(). This is incompatible with + the CancellableThreads that only allow threads to be interrupted through cancellation. The use of + CancellableThreads was introduced in #42844 and added to UnicastZenPing as part of the + backport, as both Zen1 and Zen2 share the same SeedHostsResolver implementation. This commit + effectively undoes the change in the backport while still allowing to share same implementation. + + Closes #44425 + +commit d98b3e476033da4e4559d8c32255a454e151b05a +Author: Yannick Welsch +Date: Wed Jul 17 16:53:10 2019 +0200 + + Move frozen indices to x-pack module (#44490) + + Backport of #44408 and #44286. + +commit 3a199d044c534d36f3e43b4b079ff769962a1b5b +Author: Alpar Torok +Date: Wed Jul 17 17:34:35 2019 +0300 + + Add a list of BWC versions for CI (#44418) + + This PR adds a list of index compatible versions to the `.ci` directory + as well as a way to generate and verify it. + + Unfortunetly there is no easy way in Jenkins to have the build generate then + consume this YML axis config. + I hate that this would need maintenance on versions bumps, but the + potential benefir here is reducing the bwc builds that can take more than + 24 hours to less than 20 minutes. + This is possible because the CI setup would use a matrix job to run + something like: + ``` + ./graldew 'v7.0.0#bwcTest' + ``` + For every index compatible version. + On top of that `--parallel` should be possible even without testclusters + due to the limited number of clusters being set up here. + + The example command above runs in exactly 10 minutes on my laptop, + thus I'm proposing to accept this compromise while we work out the + infra to do this more dinamically. + +commit 103ba976fd101066dfed1dfddcb339690767f8c4 +Author: Zachary Tong +Date: Wed Jul 17 10:21:56 2019 -0400 + + Convert BucketScript to static parser (#44385) + + BucketScript was using the old-style parser and could easily be + converted over to the newer static parser. + + Also adds a test for GapPolicy enum serialization + +commit 34725e20fbb83f1dcf06fc82b891f0bdaca2b11e +Author: James Rodewig +Date: Wed Jul 17 08:49:22 2019 -0400 + + [DOCS] Move Elasticsearch APIs to REST APIs section. (#44238) (#44372) + + Moves the following API sections under the REST APIs navigations: + - API Conventions + - Document APIs + - Search APIs + - Index APIs (previously named Indices APIs) + - cat APIs + - Cluster APIs + + Other supporting changes: + - Removes the previous index APIs page under REST APIs. Adds a redirect for the removed page. + - Removes several [partintro] macros so the docs build correctly. + - Changes anchors for pages that become sections of a parent page. + - Adds several redirects for existing pages that become sections of a parent page. + + This commit re-applies changes from #44238. Changes from that PR were reverted due to broken links in several repos. This commit adds redirects for those broken links. + +commit 377a6a47ac3643a3e5a62f1cd5795583d65d5f18 +Author: David Turner +Date: Wed Jul 17 13:32:59 2019 +0100 + + Improve handshake failure messages (#44485) + + Today we report an exception on a handshake failure (e.g. cluster name + mismatch) but the message does not include all the details of the mismatch. If + the mismatch is something subtle like `my-cluster` instead of `my_cluster` then + we cannot diagnose this from the message alone. This commit adds the details of + the local cluster to the message, along with the details of the remote cluster, + improving the utility of the exception message if reported in isolation. + +commit 91673e373acaf2d2cd0be0c4ae85d320f5343627 +Author: Armin Braun +Date: Wed Jul 17 14:31:46 2019 +0200 + + Fix Incorrect Uncompressed Error Handling in InboundMessage (#44317) (#44483) + + * Fix Incorrect Uncompressed Error Handling in InboundMessage + + * CompressorFactory.compressor does not throw uncompressed exception on uncompressed bytes, it merely returns `null` in this case if the bytes are at least XContent so the current catch and re-throw logic is dead code + * Made it work again by throwing on a `null` return so we get a real error message instead of an NPE + +commit eb348d2593b6d99424619bd3b5455736d4ac9e5c +Author: Ignacio Vera +Date: Wed Jul 17 13:28:28 2019 +0200 + + Upgrade to lucene-8.2.0-snapshot-6413aae226 (#44480) + +commit c8db0e9b7edfad502b56294be90bd6e65022478b +Author: Armin Braun +Date: Wed Jul 17 11:56:02 2019 +0200 + + Remove blobExists Method from BlobContainer (#44472) (#44475) + + * We only use this method in one place in production code and can replace that with a read -> remove it to simplify the interface + * Keep it as an implementation detail in the Azure repository + +commit e423b7341aefb4176571e284858d25619ced4416 +Author: Tanguy Leroux +Date: Wed Jul 17 10:30:30 2019 +0200 + + Log non-acknowledged close index response in ReplicaToPrimaryPromotionIT + + Relates #44479 + +commit dca8a918f336f23a9f9c0613f28e584eba951edb +Author: David Turner +Date: Wed Jul 17 08:34:56 2019 +0100 + + Use applied cluster state in cluster health (#44426) + + In #44348 we changed the cluster health action so that it sometimes uses the + cluster state directly from the master service rather than from the cluster + applier. If the state is not recovered then this is inappropriate, because + prior to state recovery the state available to the cluster applier contains no + indices. This commit moves us back to using the state from the applier. + + Fixes #44416. + +commit 0fd33b089f3491e9b0b58e6105c17d91dd9b5c7d +Author: David Turner +Date: Wed Jul 17 08:33:16 2019 +0100 + + Report shard state changes better (#44419) + + Today when the cluster health changes the `AllocationService` reports at most + ten shards that were started or failed, and always ends its message with `...` + suggesting that the list is truncated. This commit adjusts these messages to be + clearer about whether the list is truncated or not. When debug logging is + enabled the list is not truncated; if the list is truncated then its length is + logged, and if it is not truncated then no `...` is included in the message. + +commit 6e50bafa8f4b60bd5fedda636059ab0ba375328f +Author: Ryan Ernst +Date: Tue Jul 16 23:24:02 2019 -0700 + + Convert Broadcast request and response to use writeable.reader (#44386) (#44453) + + This commit converts the request and response classes for broadcast + actions to implement ctors for Writeable.Reader and forces all future + implementations to implement the same. + + relates #34389 + +commit 6b1a7696387da57e8aaf6ce12df7ca90f662b9e6 +Author: Tim Brooks +Date: Wed Jul 3 09:06:23 2019 -0400 + + Move CORS Config into :server package (#43779) + + This commit moves the config that stores Cors options into the server + package. Currently both nio and netty modules must have a copy of this + config. Moving it into server allows one copy and the tests to be in a + common location. + +commit 0a352486e8f18fbc0da062a3d021ddead9c9f486 +Author: Tim Brooks +Date: Tue Jul 16 18:46:41 2019 -0400 + + Isolate nio channel registered from channel active (#44388) + + Registering a channel with a selector is a required operation for the + channel to be handled properly. Currently, we mix the registeration with + other setup operations (ip filtering, SSL initiation, etc). However, a + fail to register is fatal. This PR modifies how registeration occurs to + immediately close the channel if it fails. + + There are still two clear loopholes for how a user can interact with a + channel even if registration fails. 1. through the exception handler. + 2. through the channel accepted callback. These can perhaps be improved + in the future. For now, this PR prevents writes from proceeding if the + channel is not registered. + +commit cc0ff3aa71a0f4002d1cbf249f330a1c5aacd02d +Author: Julie Tibshirani +Date: Tue Jul 16 14:43:04 2019 -0700 + + Ensure field caps doesn't error on rank feature fields. (#44370) + + The contract for MappedFieldType#fielddataBuilder is to throw an + IllegalArgumentException if fielddata is not supported. The rank feature mappers + were instead throwing an UnsupportedOperationException, which caused + MappedFieldType#isAggregatable to fail. + +commit 100cb89f3e40ce307bba3070aca447498a24a48d +Author: Jason Tedor +Date: Wed Jul 17 07:23:40 2019 +0900 + + Avoid stack overflow in auto-follow coordinator (#44421) + + This commit avoids a situation where we might stack overflow in the + auto-follower coordinator. In the face of repeated failures to get the + remote cluster state, we would previously be called back on the same + thread and then recurse to try again. If this failure persists, the + repeated callbacks on the same thread would lead to a stack + overflow. The repeated failures can occur, for example, if the connect + queue is full when we attempt to make a connection to the remote + cluster. This commit avoids this by truncating the call stack if we are + called back on the same thread as the initial request was made on. + +commit c18a6402cd5971fe5d164a684b61a6217e8ed03c +Author: Lisa Cawley +Date: Tue Jul 16 15:09:00 2019 -0700 + + [DOCS] Renames data frame APIs section (#44458) + +commit becbf450fa256fd04b76a4efda58495f6d464e5d +Author: Jason Tedor +Date: Wed Jul 17 06:24:24 2019 +0900 + + Avoid NPE when checking for CCR index privileges (#44397) + + This commit avoids an NPE when checking for privileges to follow + indices. The problem here is that in some cases we might not be able to + read the authentication info from the thread context. In that case, a + null user would be returned and we were not guarding against this. + +commit 53514b0477467403d132181b09394122a766ad44 +Author: Lisa Cawley +Date: Tue Jul 16 13:22:27 2019 -0700 + + [DOCS] Separates data frame analytics APIs (#44451) + +commit 3220709b0a6ae2bfb8c701fc13054bd0f0918930 +Author: Mayya Sharipova +Date: Tue Jul 16 16:22:11 2019 -0400 + + Add positions info into term_vector doc (#44379) + +commit c26edb4c436b24a652c76caed07c1042cdf733b4 +Author: Ryan Ernst +Date: Tue Jul 16 12:53:08 2019 -0700 + + Ensure replication response/requests implement writeable (#44392) (#44446) + + This commit cleans up replication response and request so that the base + class does not allow subclasses to implement Streamable. + + relates #34389 + +commit 901310a8268fa8034ce224f43e3a97c903e9af6c +Author: Tal Levy +Date: Tue Jul 16 12:41:10 2019 -0700 + + [7.x] Migrate ML Actions to use writeable ActionType (#44302) (#44391) + + * Migrate ML Actions to use writeable ActionType (#44302) + + This commit converts all the StreamableResponseActionType + actions in the ML core module to be ActionType and leverage + the Writeable infrastructure. + +commit 642e9019ffdaae2bdec424c7dd8c535f33aa4949 +Author: István Zoltán Szabó +Date: Tue Jul 16 18:36:55 2019 +0200 + + [DOCS] Adds max_page_search_size to data frame transform pivot properties (#44126) + +commit 9613700a631ea6db81bc1e9567eeba9255506a7a +Author: Przemysław Witek +Date: Tue Jul 16 21:22:40 2019 +0200 + + [7.x] Implement MlConfigIndexMappingsFullClusterRestartIT test which verifies that .ml-config index mappings are properly updated during cluster upgrade (#44341) (#44366) + +commit ac07eef86c74b48af1d53e48888e472f409dbcec +Author: James Rodewig +Date: Tue Jul 16 15:02:38 2019 -0400 + + [DOCS] Remove :edit_url: overrides. (#44445) + + These overrides do not work in Asciidoctor and are no longer needed. + +commit 301c8daf4ca260306d68df7b51347ba9d8c569a9 +Author: Nhat Nguyen +Date: Tue Jul 16 14:18:35 2019 -0400 + + Revert "Make peer recovery send file chunks async (#44040)" + + This reverts commit a2b4687d893a8076607146fbd826c7210e85c538. + +commit a3ba11cfc13e18bd286d41e995089088280afe92 +Author: James Baiera +Date: Tue Jul 16 13:58:57 2019 -0400 + + Improve CryptoService error message on missing secure file (#43623) (#44364) + + This improves the error message when encrypting of sensitive watcher + data is configured, but no system file was specified in the keystore. + This error message is displayed on startup. + + This also closes the input stream of the secure file properly. + + Closes #43619 + +commit 67ec0a4e9bf83cfd2da70be27a726cbea949712f +Author: Christoph Büscher +Date: Tue Jul 16 19:41:59 2019 +0200 + + Unmute SpecificMasterNodesIT test (#44436) + + The underlying issue is closed and the fix in #42454 seems to have been + backported to 7.x and 7.3 so we can reactivate the test. + +commit 563a78829fc2d87f5f4cb9246a0bedf9f38866e5 +Author: Yu +Date: Wed Jul 17 00:53:47 2019 +0800 + + Do not allow version in Rest Update API (#43516) + + The versioning of Update API doesn't rely on version number anymore (and + rather on sequence number). But in rest api level we ignored the + "version" and "version_type" parameter, so that the server cannot raise + the exception when whey were set. + + This PR restores "version" and "version_type" parsing in Update Rest API + so that we can get the appropriate errors. + + Relates to #42497 + +commit 5ea89a4e7ba7e9e441fa1027c9f1131ead52fabb +Author: Hendrik Muhs +Date: Tue Jul 16 14:05:49 2019 +0200 + + Remove PR 41491 from release notes (#44417) + + PR 41491 hasn't been merged into 7.2 but got closed and superseeded by #41800 (merged via feature branch merge into 7.3) + +commit 54194021bf73cd58aa5b161dea5644689dfe1c00 +Author: Mark Vieira +Date: Tue Jul 16 09:34:00 2019 -0700 + + Improve build scan metadata (#44247) + + (cherry picked from commit 2797b2452b1d3696a4c3a367f21630306da5c818) + +commit 2c7ff812da49acb5e0cacf60588d523a07dc9dd6 +Author: Benjamin Trent +Date: Tue Jul 16 11:11:31 2019 -0500 + + [ML] Add r_squared eval metric to regression (#44248) (#44378) + + * [ML] Add r_squared eval metric to regression + + * fixing tests and binarysoftclassification class + + * Update RSquared.java + + * Update x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquared.java + + Co-Authored-By: David Kyle + + * removing unnecessary debug test + +commit 858dbfc0746cf72d156f14898a47f166af98d7fd +Author: Benjamin Trent +Date: Tue Jul 16 10:04:28 2019 -0500 + + [ML][Data Frame] treat bulk index failures as an indexing failure (#44351) (#44427) + + * [ML][Data Frame] treat bulk index failures as an indexing failure + + * removing redundant public modifier + + * changing to an ElasticsearchException + + * fixing redundant public modifier + +commit a2b4687d893a8076607146fbd826c7210e85c538 +Author: Nhat Nguyen +Date: Tue Jul 16 09:52:51 2019 -0400 + + Make peer recovery send file chunks async (#44040) + +commit 34bf6bcec0666a20d12eabd5bd0c52e32cc71507 +Author: Przemysław Witek +Date: Tue Jul 16 16:15:32 2019 +0200 + + Treat big changes in searchCount as significant and persist the document after such changes (#44413) (#44424) + +commit eb7d43f4cfe910bd06d107176866494c2f7df5cd +Author: Jake Landis +Date: Tue Jul 16 08:48:09 2019 -0500 + + Log write failures for watcher history document. (#44129) (#44357) + + The failure is correctly getting propagated, this commit adds support to + explicitly look for .watch-history failures using the same logging strategy + as triggered watch failures. + +commit fb0461ac7672bef35447cbdefafb21e362599163 +Author: Lee Hinman +Date: Tue Jul 16 07:37:13 2019 -0600 + + [7.x] Add Snapshot Lifecycle Management (#44382) + + * Add Snapshot Lifecycle Management (#43934) + + * Add SnapshotLifecycleService and related CRUD APIs + + This commit adds `SnapshotLifecycleService` as a new service under the ilm + plugin. This service handles snapshot lifecycle policies by scheduling based on + the policies defined schedule. + + This also includes the get, put, and delete APIs for these policies + + Relates to #38461 + + * Make scheduledJobIds return an immutable set + + * Use Object.equals for SnapshotLifecyclePolicy + + * Remove unneeded TODO + + * Implement ToXContentFragment on SnapshotLifecyclePolicyItem + + * Copy contents of the scheduledJobIds + + * Handle snapshot lifecycle policy updates and deletions (#40062) + + (Note this is a PR against the `snapshot-lifecycle-management` feature branch) + + This adds logic to `SnapshotLifecycleService` to handle updates and deletes for + snapshot policies. Policies with incremented versions have the old policy + cancelled and the new one scheduled. Deleted policies have their schedules + cancelled when they are no longer present in the cluster state metadata. + + Relates to #38461 + + * Take a snapshot for the policy when the SLM policy is triggered (#40383) + + (This is a PR for the `snapshot-lifecycle-management` branch) + + This commit fills in `SnapshotLifecycleTask` to actually perform the + snapshotting when the policy is triggered. Currently there is no handling of the + results (other than logging) as that will be added in subsequent work. + + This also adds unit tests and an integration test that schedules a policy and + ensures that a snapshot is correctly taken. + + Relates to #38461 + + * Record most recent snapshot policy success/failure (#40619) + + Keeping a record of the results of the successes and failures will aid + troubleshooting of policies and make users more confident that their + snapshots are being taken as expected. + + This is the first step toward writing history in a more permanent + fashion. + + * Validate snapshot lifecycle policies (#40654) + + (This is a PR against the `snapshot-lifecycle-management` branch) + + With the commit, we now validate the content of snapshot lifecycle policies when + the policy is being created or updated. This checks for the validity of the id, + name, schedule, and repository. Additionally, cluster state is checked to ensure + that the repository exists prior to the lifecycle being added to the cluster + state. + + Part of #38461 + + * Hook SLM into ILM's start and stop APIs (#40871) + + (This pull request is for the `snapshot-lifecycle-management` branch) + + This change allows the existing `/_ilm/stop` and `/_ilm/start` APIs to also + manage snapshot lifecycle scheduling. When ILM is stopped all scheduled jobs are + cancelled. + + Relates to #38461 + + * Add tests for SnapshotLifecyclePolicyItem (#40912) + + Adds serialization tests for SnapshotLifecyclePolicyItem. + + * Fix improper import in build.gradle after master merge + + * Add human readable version of modified date for snapshot lifecycle policy (#41035) + + * Add human readable version of modified date for snapshot lifecycle policy + + This small change changes it from: + + ``` + ... + "modified_date": 1554843903242, + ... + ``` + + To + + ``` + ... + "modified_date" : "2019-04-09T21:05:03.242Z", + "modified_date_millis" : 1554843903242, + ... + ``` + + Including the `"modified_date"` field when the `?human` field is used. + + Relates to #38461 + + * Fix test + + * Add API to execute SLM policy on demand (#41038) + + This commit adds the ability to perform a snapshot on demand for a policy. This + can be useful to take a snapshot immediately prior to performing some sort of + maintenance. + + ```json + PUT /_ilm/snapshot//_execute + ``` + + And it returns the response with the generated snapshot name: + + ```json + { + "snapshot_name" : "production-snap-2019.04.09-rfyv3j9qreixkdbnfuw0ug" + } + ``` + + Note that this does not allow waiting for the snapshot, and the snapshot could + still fail. It *does* record this information into the cluster state similar to + a regularly trigged SLM job. + + Relates to #38461 + + * Add next_execution to SLM policy metadata (#41221) + + * Add next_execution to SLM policy metadata + + This adds the next time a snapshot lifecycle policy will be executed when + retriving a policy's metadata, for example: + + ```json + GET /_ilm/snapshot?human + { + "production" : { + "version" : 1, + "modified_date" : "2019-04-15T21:16:21.865Z", + "modified_date_millis" : 1555362981865, + "policy" : { + "name" : "", + "schedule" : "*/30 * * * * ?", + "repository" : "repo", + "config" : { + "indices" : [ + "foo-*", + "important" + ], + "ignore_unavailable" : true, + "include_global_state" : false + } + }, + "next_execution" : "2019-04-15T21:16:30.000Z", + "next_execution_millis" : 1555362990000 + }, + "other" : { + "version" : 1, + "modified_date" : "2019-04-15T21:12:19.959Z", + "modified_date_millis" : 1555362739959, + "policy" : { + "name" : "", + "schedule" : "0 30 2 * * ?", + "repository" : "repo", + "config" : { + "indices" : [ + "other" + ], + "ignore_unavailable" : false, + "include_global_state" : true + } + }, + "next_execution" : "2019-04-16T02:30:00.000Z", + "next_execution_millis" : 1555381800000 + } + } + ``` + + Relates to #38461 + + * Fix and enhance tests + + * Figured out how to Cron + + * Change SLM endpoint from /_ilm/* to /_slm/* (#41320) + + This commit changes the endpoint for snapshot lifecycle management from: + + ``` + GET /_ilm/snapshot/ + ``` + + to: + + ``` + GET /_slm/policy/ + ``` + + It mimics the ILM path only using `slm` instead of `ilm`. + + Relates to #38461 + + * Add initial documentation for SLM (#41510) + + * Add initial documentation for SLM + + This adds the initial documentation for snapshot lifecycle management. + + It also includes the REST spec API json files since they're sort of + documentation. + + Relates to #38461 + + * Add `manage_slm` and `read_slm` roles (#41607) + + * Add `manage_slm` and `read_slm` roles + + This adds two more built in roles - + + `manage_slm` which has permission to perform any of the SLM actions, as well as + stopping, starting, and retrieving the operation status of ILM. + + `read_slm` which has permission to retrieve snapshot lifecycle policies as well + as retrieving the operation status of ILM. + + Relates to #38461 + + * Add execute to the test + + * Fix ilm -> slm typo in test + + * Record SLM history into an index (#41707) + + It is useful to have a record of the actions that Snapshot Lifecycle + Management takes, especially for the purposes of alerting when a + snapshot fails or has not been taken successfully for a certain amount of + time. + + This adds the infrastructure to record SLM actions into an index that + can be queried at leisure, along with a lifecycle policy so that this + history does not grow without bound. + + Additionally, + SLM automatically setting up an index + lifecycle policy leads to + `index_lifecycle` custom metadata in the cluster state, which some of + the ML tests don't know how to deal with due to setting up custom + `NamedXContentRegistry`s. Watcher would cause the same problem, but it + is already disabled (for the same reason). + + * High Level Rest Client support for SLM (#41767) + + * High Level Rest Client support for SLM + + This commit add HLRC support for SLM. + + Relates to #38461 + + * Fill out documentation tests with tags + + * Add more callouts and asciidoc for HLRC + + * Update javadoc links to real locations + + * Add security test testing SLM cluster privileges (#42678) + + * Add security test testing SLM cluster privileges + + This adds a test to `PermissionsIT` that uses the `manage_slm` and `read_slm` + cluster privileges. + + Relates to #38461 + + * Don't redefine vars + + * Add Getting Started Guide for SLM (#42878) + + This commit adds a basic Getting Started Guide for SLM. + + * Include SLM policy name in Snapshot metadata (#43132) + + Keep track of which SLM policy in the metadata field of the Snapshots + taken by SLM. This allows users to more easily understand where the + snapshot came from, and will enable future SLM features such as + retention policies. + + * Fix compilation after master merge + + * [TEST] Move exception wrapping for devious exception throwing + + Fixes an issue where an exception was created from one line and thrown in another. + + * Fix SLM for the change to AcknowledgedResponse + + * Add Snapshot Lifecycle Management Package Docs (#43535) + + * Fix compilation for transport actions now that task is required + + * Add a note mentioning the privileges needed for SLM (#43708) + + * Add a note mentioning the privileges needed for SLM + + This adds a note to the top of the "getting started with SLM" + documentation mentioning that there are two built-in privileges to + assist with creating roles for SLM users and administrators. + + Relates to #38461 + + * Mention that you can create snapshots for indices you can't read + + * Fix REST tests for new number of cluster privileges + + * Mute testThatNonExistingTemplatesAreAddedImmediately (#43951) + + * Fix SnapshotHistoryStoreTests after merge + + * Remove overridden newResponse functions that have been removed + + * Fix compilation for backport + + * Fix get snapshot output parsing in test + + * [DOCS] Add redirects for removed autogen anchors (#44380) + + * Switch ... in javadocs for {@code ...} + +commit aa9dd313cf658dc787d65e01b68d6b9309c05c90 +Author: Lucas Groenendaal +Date: Tue Jul 16 07:10:06 2019 -0500 + + Fix incorrect node name in docs (#43062) + + After starting up elasticsearch the documentation said that their node + name was "6-bjhwl" but in the documentation's output I did not see that + node name. Instead I saw the node name as `localhost.localdomain` + +commit ae5a917efe336836888863cf7ae7324f950b2fe4 +Author: david raistrick +Date: Mon Jul 15 09:44:34 2019 -0400 + + Add clarification around TESTSETUP docs and error message (#43306) + +commit 4a5215d22aa14cac657d0acc71c2e3926554ed17 +Author: Mark Walkom +Date: Mon Jul 15 23:39:20 2019 +1000 + + [DOCS] Update id-field.asciidoc (#42482) + + Adding a note around the size limit for `_id` + +commit 8a2d23671a66aba317feb1fbddd716a3155c91ac +Author: Dan Fey +Date: Mon Jul 15 05:27:30 2019 -0700 + + [DOCS] Update split-index.asciidoc: fix shards example (#41382) + + The max value should be 640 instead of 740 in the shard example: + +commit 078efc9ec47f295ee81c87b381d6838330d34a16 +Author: Tanguy Buchier +Date: Mon Jul 15 17:26:56 2019 +0800 + + [DOCS] Clarify refresh_interval new behavior (#43726) + + Update indexing-speed.asciidoc to clarify refresh_interval new behavior + +commit 6c1f7407596ba783bb38175561b9baec1b8dfe35 +Author: Hendrik Muhs +Date: Tue Jul 16 13:43:13 2019 +0200 + + [ML-DataFrame] make checkpointing more robust (#44344) (#44414) + + make checkpointing more robust: + + - do not let checkpointing fail if indexes got deleted + - treat missing seqNoStats as just created indices (checkpoint 0) + - loglevel: do not treat failed updated checks as error + + fixes #43992 + +commit 096c03945c71b5a50fb2e5d591b1bf67bca5b703 +Author: magnusram05 <44857541+magnusram05@users.noreply.github.com> +Date: Tue Jul 16 17:08:42 2019 +0530 + + [Docs] Small update to getting-started.asciidoc (#40393) + +commit 3f3a3d3f2b847557619bd99da75d969b15b07ece +Author: Przemysław Witek +Date: Tue Jul 16 12:51:29 2019 +0200 + + [7.x] Add DatafeedTimingStats.average_search_time_per_bucket_ms and TimingStats.total_bucket_processing_time_ms stats (#44125) (#44404) + +commit 4a79ccd3246409c699ef1ffc906d68106c3fbe2b +Author: Armin Braun +Date: Tue Jul 16 12:20:52 2019 +0200 + + Cleaner Exception Handling on Shard Delete (#44384) (#44407) + + * Follow up to #44165 + * We should just catch all exceptions here and not return errors after the index-N update went through since a subsequent delete attempt by the user would fail with SnapshotMissingException since the snapshot now appears deleted. Also, `SnapshotException` isn't even thrown in the changed spot it seems in the first place and certainly not the only exception possible. + +commit 940aa71930e359c3018af02b967a4c7fab50782d +Author: Armin Braun +Date: Tue Jul 16 12:19:20 2019 +0200 + + Cleanup S3 BlobContainer Listing Logic (#43088) (#44406) + + * Cleanup duplication in creating and looping over IO Requests + +commit a09389c5113ecd9d8dcd5d34611b42b733a551b2 +Author: David Turner +Date: Tue Jul 16 11:00:26 2019 +0100 + + AwaitsFix GatewayIndexStateIT#testJustMasterNode + + Relates #44416. + +commit c4cf98c53804cd057e796a629435a8174d65fbfc +Author: Ryan Ernst +Date: Tue Jul 16 01:11:13 2019 -0700 + + Convert core security actions to use writeable ActionType (#44359) (#44390) + + This commit converts all the StreamableResponseActionType security + classes in xpack core to ActionType, implementing Writeable for their + response classes. + + relates #34389 + +commit be98a12cd040785fe77b4bceb379fef3c73ee827 +Author: Jason Tedor +Date: Tue Jul 16 16:12:36 2019 +0900 + + Do not swallow I/O exception getting authentication (#44398) + + When getting authentication info from the thread context, it might be + that we encounter an I/O exception. Today we swallow this exception and + return a null authentication info to the caller. Yet, this could be + hiding bugs or errors. This commits adjusts this behavior so that we no + longer swallow the exception. + +commit 4b50de2e2e5e7cd2b3fc295a11bbc090030f0773 +Author: Tim Vernum +Date: Tue Jul 16 16:22:25 2019 +1000 + + Document xpack.security.dls.bitset.cache settings (#44400) + + Two new settings were introduced in #43669 to control the + behaviour of the Document Level Security BitSet cache. + + This change adds documentation for these 2 settings. + + Backport of: #44100 + +commit 673c63bb0029adcbd8e1a8dff7d597e1fccedfe8 +Author: William Brafford +Date: Tue Jul 16 01:48:12 2019 -0400 + + Restrict testCreateEmptyDirNoPermissions to Unix (#44282) (#44297) + + The test EmptyDirTaskTests#testCreateEmptyDirNoPermissions may fail on + Windows. However, the test is only meaningful for Unix permissions + structures, so we should assume a Unix-family OS and skip the test on + Windows. + + Fixes #44064 + +commit 8d68d1f54dbaf90116d9990202744fb63d59321f +Author: David Turner +Date: Tue Jul 16 06:31:15 2019 +0100 + + Cluster health should await events plus other things (#44348) + + Today a cluster health request can wait on a selection of conditions, but it + does not guarantee that all of these conditions have ever held simultaneously + when it returns. More specifically, if a request sets `waitForEvents()` along + with some other conditions then Elasticsearch will respond when the master has + processed all the expected pending tasks _and then_ the cluster satisfied the + other conditions, but it may be that at the time the cluster satisfied the + other conditions there were undesired pending tasks again. + + This commit adjusts the behaviour of `waitForEvents()` to wait for all the + required events to be processed and then, if the resulting cluster state does + not satisfy the other conditions, it will wait until there is a cluster state + that does and then retry the wait-for-events too. + +commit 5c8275cd2cbcca26d37a83c433e418840bb0807f +Author: Armin Braun +Date: Tue Jul 16 07:06:26 2019 +0200 + + Fix Exceptions in EventHandler#postHandling Breaking Select Loop (#44347) (#44396) + + * Fix Exceptions in EventHandler#postHandling Breaking Select Loop + + * We can run into the `write` path for SSL channels when they are not fully registered (if registration fails and a close message is attempted to be written) and thus into NPEs from missing selection keys + * This is a quick fix to quiet down tests, a cleaner solution will be incoming for #44343 + * Relates #44343 + +commit 099d52f3b0b0f3baaaaa6846d23ef2c3f7bf9891 +Author: Armin Braun +Date: Tue Jul 16 04:40:50 2019 +0200 + + Prevent Confusing Blocked Thread Warnings in MockNioTransport (#44356) (#44376) + + * Prevent Confusing Blocked Thread Warnings in MockNioTransport + + * We can run into a race where the stacktrace collection and subsequent logging happens after the thread has already unblocked thus logging a confusing stacktrace of wherever the transport thread was after it became unblocked + * Fixed this by comparing whether or not the recorded timestamp is still the same before and after the stacktrace was recorded and not logging if it already changed + +commit e0b82e92f335eec9fb53e168092939db2ae94327 +Author: Ryan Ernst +Date: Mon Jul 15 18:07:52 2019 -0700 + + Convert BaseNode(s) Request/Response classes to Writeable (#44301) (#44358) + + This commit converts all BaseNodeResponse and BaseNodesResponse + subclasses to implement Writeable.Reader instead of Streamable. + + relates #34389 + +commit 7e06888bae01b138604faf37c6294ab857231b1e +Author: Ryan Ernst +Date: Mon Jul 15 17:53:05 2019 -0700 + + Convert testclusters to use distro download plugin (#44253) (#44362) + + Test clusters currently has its own set of logic for dealing with + finding different versions of Elasticsearch, downloading them, and + extracting them. This commit converts testclusters to use the + DistributionDownloadPlugin. + +commit 753da8feac33e3dc38241f64a82b51d2dfa4aa5a +Author: Lisa Cawley +Date: Mon Jul 15 13:48:04 2019 -0700 + + [DOCS] Updates terminology for alerting features (#43945) + +commit c00b0827011d089a099e9f49767c8ce3a2aad800 +Author: Jake Landis +Date: Mon Jul 15 15:01:43 2019 -0500 + + add 7.2.1 release notes (#44367) + +commit a848fc9bf47b387027e6134f320d19cc435f8a5a +Author: Yannick Welsch +Date: Mon Jul 15 21:41:25 2019 +0200 + + Revert "Add usage stats for frozen indices (#44286)" + + This reverts commit 5e73c49ec8ed715988b1ce8ac6e0c17d1b9af463. + +commit 7b68bfb4e66acfa10a2de5ab640bde9a410626d5 +Author: Yannick Welsch +Date: Mon Jul 15 21:41:21 2019 +0200 + + Revert "Add frozen indices usage for all but transport client (#44286)" + + This reverts commit d2d40afc02bd03b950214819f5d1cb11f11add3e. + +commit d2d40afc02bd03b950214819f5d1cb11f11add3e +Author: Yannick Welsch +Date: Mon Jul 15 20:47:39 2019 +0200 + + Add frozen indices usage for all but transport client (#44286) + + Backport gone wrong. + +commit 373435695593989fbe57ef552c0d053f44c2be2b +Author: Adrien Grand +Date: Mon Jul 15 20:00:41 2019 +0200 + + Update release notes. + +commit e7ea49e32ff2e15698b2553fb01b93971ed7a5fd +Author: Lisa Cawley +Date: Mon Jul 15 09:44:57 2019 -0700 + + [DOCS] Removes unnecessary resource definition pages (#44289) + +commit 141d09ee154e9395eb7b292a504422b5d8edba61 +Author: Julie Tibshirani +Date: Mon Jul 15 09:44:25 2019 -0700 + + Correct a formatting mistake in the _field_caps docs. (#44303) + + The 'indices' block that was recently added should appear in the top-level of + the response, as opposed to being nested under 'fields'. + +commit 86ee8eab3f01799b519b5425e362538b5ab4f570 +Author: David Turner +Date: Mon Jul 15 17:37:31 2019 +0100 + + Allow RerouteService to reroute at lower priority (#44338) + + Today the `BatchedRerouteService` submits its delayed reroute task at `HIGH` + priority, but in some cases a lower priority would be more appropriate. This + commit adds the facility to submit delayed reroute tasks at different + priorities, such that each submitted reroute task runs at a priority no lower + than the one requested. It does not change the fact that all delayed reroute + tasks are submitted at `HIGH` priority, but at least it makes this explicit. + +commit 6c7f7d4a10765bc274646d944a31db50e09f7501 +Author: Lisa Cawley +Date: Mon Jul 15 09:11:04 2019 -0700 + + [DOCS] Adds ml-cpp PRs to release notes (#44354) + +commit 59658daef94face9a7d9c30b0629c7f0f043b0ea +Author: Ryan Ernst +Date: Mon Jul 15 09:20:20 2019 -0700 + + Separate streamable based master node actions (#44313) + + This commit creates new base classes for master node actions whose + response types still implement Streamable. This simplifies both finding + remaining classes to convert, as well as creating new master node + actions that use Writeable for their responses. + + relates #34389 + +commit 5e73c49ec8ed715988b1ce8ac6e0c17d1b9af463 +Author: Yannick Welsch +Date: Mon Jul 15 16:24:00 2019 +0200 + + Add usage stats for frozen indices (#44286) + + Adds usage stats for frozen indices of the form: + + "frozen_indices" : { + "available" : true, + "enabled" : true, + "indices_count" : 0 + } + +commit e3d2af64c4e9151f7faa2f4d3104c98b1de7a428 +Author: David Turner +Date: Mon Jul 15 15:19:54 2019 +0100 + + Throw TranslogCorruptedException in more cases (#44217) + + Today we do not throw a `TranslogCorruptedException` in certain cases of + translog corruption, such as for a corrupted checkpoint file or when an + expected file (either checkpoint or translog) is completely missing. This means + that `elasticsearch-shard` will not truncate the translog in those cases. + + This commit strengthens the translog corruption tests to corrupt and/or delete + both translog and checkpoint files, and ensures that a + `TranslogCorruptedException` is thrown in all cases. It also sometimes + simulates a recovery after a crash while rolling the translog generation, + including cases where the rolled checkpoint contains incorrect data. + + It also adjusts (and renames) `RemoveCorruptedShardDataCommandIT.getDirs()` to + return only a single path, since in practice this was the only thing that could + happen and yet we were relying on its callers to verify this and not all + callers were doing so. + +commit 238270154748ba8cae666efa6e662d33fb5098d3 +Author: David Kyle +Date: Mon Jul 15 11:58:09 2019 +0100 + + Wait for pending tasks in docs tests cleanup (#44123) + + ML and Data Frame tests should wait for pending tasks + +commit eb1106c465c8b73c260f5ecc26458a4a7da89509 +Author: Armin Braun +Date: Mon Jul 15 12:59:38 2019 +0200 + + Stronger Cleanup Shard Snapshot Directory on Delete (#44257) (#44337) + + * Stronger Cleanup Shard Snapshot Directory on Delete + + * Use `RepositoryData` to clean up unreferenced `snap-${uuid}.dat` blobs + from shard directories (and index-N) and as a result also clean up data + blobs that are only referenced by them + * Stop cleaning up anything but index-N on shard snapshot creation to + align behavior of shard index-N handling with root path index-N + handling + +commit 22dc125dad6fd1db99dc30318046fd19bb3635dc +Author: Christoph Büscher +Date: Mon Jul 15 11:52:38 2019 +0200 + + AnalyzeAction.Response doesn't need to call super.readFrom() (#44331) + + The responses super.writeTo() method was removed in #44092, so the corresponding + contructor that reads from a stream shouldn't call super itself, even though its + implementation is currently empty. + +commit 7f5d40d23560dcdba2aaf545d99283644eb3d842 +Author: Armin Braun +Date: Mon Jul 15 10:59:51 2019 +0200 + + Avoid Needless Set Instantiation in InboundMessage (#44318) (#44329) + + * Avoid Needless Set Instantiation in InboundMessage + + * When `features` is empty (when there's no xpack) we constantly and needless instantiated a few objects here for the empty set on every message + +commit 0cc94a457d9f5fbfcc764321f1cd341e92727412 +Author: Armin Braun +Date: Mon Jul 15 10:59:33 2019 +0200 + + Remove non-SMILE Serialization from ChecksumBlobStoreFormat (#44278) (#44326) + + * At least all the way back to 6.x we never use anything but `SMILE` in + production code with this class so I removed the more general + constructor and removed the format leniency from the deserialization + +commit 76a96c37742c39c139ff18e65b955e21ca480908 +Author: Tanguy Leroux +Date: Mon Jul 15 10:20:29 2019 +0200 + + Remove ReusePeerRecoverySharedTest class (#44275) + +commit d73e2f9c5601462e63e7bb7f608e1b352b275e6b +Author: Armin Braun +Date: Mon Jul 15 10:21:54 2019 +0200 + + HLRC: Fix '+' Not Correctly Encoded in GET Req. (#33164) (#44324) + + * HLRC: Fix '+' Not Correctly Encoded in GET Req. + + * Encode `+` correctly as `%2B` in URL paths + * Keep encoding `+` as space in URL parameters + * Closes #33077 + +commit fc6a31e141b0e0b720e88c2456c48bba25c336ab +Author: Ryan Ernst +Date: Sun Jul 14 19:04:53 2019 -0700 + + Use specific version constant for wire bwc check (#44316) + + This commit modifies bwc behavior in FindFileStructureAction to check + against a concrete version instead of Version.CURRENT. Checking against + Version.CURRENT does not work since it is changing, in addition to it + having different meanings on each branch. + + relates #42501 + +commit 2203d447aa26173364af6da5d8ab8303b45deb07 +Author: Nhat Nguyen +Date: Sun Jul 14 19:25:11 2019 -0400 + + Fail engine if hit document failure on replicas (#43523) + + An indexing on a replica should never fail after it was successfully + indexed on a primary. Hence, we should fail an engine if we hit any + failure (document level or tragic failure) when processing an indexing + on a replica. + + Relates #43228 + Closes #40435 + +commit 835b7a120d5232a09f8354996380149f96b01cca +Author: Christoph Büscher +Date: Fri Jul 12 21:50:39 2019 +0200 + + Fix AnalyzeAction response serialization (#44284) + + Currently we loose information about whether a token list in an AnalyzeAction + response is null or an empty list, because we write a 0 value to the stream in + both cases and deserialize to a null value on the receiving side. This change + fixes this so we write an additional flag indicating whether the value is null + or not, followed by the size of the list and its content. + + Closes #44078 + +commit b40b6dd5425f212c1b480f028d731dbf1f36e5b8 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sat Jul 13 10:07:40 2019 +1000 + + Disable repository-hdfs tests in FIPS jvm (#44283) + + Due to https://github.com/elastic/elasticsearch/issues/40079, + we need to disable repository-hdfs tests in FIPS jvm. + +commit 33627ef41047426dbfe3d88c74e5192a1578dcda +Author: Hendrik Muhs +Date: Sat Jul 13 07:11:45 2019 +0200 + + re-enable bwc tests + +commit 684b562381775cf860556b5d28995c10dc2a7aca +Author: Hendrik Muhs +Date: Sat Jul 13 06:58:04 2019 +0200 + + [7.x][ML-DataFrame] Rewrite continuous logic to prevent terms count limit (#44287) + + Rewrites how continuous data frame transforms calculates and handles buckets that require an update. Instead of storing the whole set in memory, it pages through the updates using a 2nd cursor. This lowers memory consumption and prevents problems with limits at query time (max_terms_count). The list of updates can be re-retrieved in a failure case (#43662) + +commit 1dcf53465ce0dd08a7d75c94658c01ed636067a0 +Author: Ryan Ernst +Date: Fri Jul 12 13:08:10 2019 -0700 + + Reorder HandledTransportAction ctor args (#44291) + + This commit moves the Supplier variant of HandledTransportAction to have + a different ordering than the Writeable.Reader variant. The Supplier + version is used for the legacy Streamable, and currently having the + location of the Writeable.Reader vs Supplier in the same place forces + using casts of Writeable.Reader to select the correct super constructor. + This change in ordering allows easier migration to Writeable.Reader. + + relates #34389 + +commit 51ff6b420a9ebfa4569c59a0d5cdbcb22721b33e +Author: Benjamin Trent +Date: Fri Jul 12 15:24:49 2019 -0500 + + [ML][Data Frame] prevent task from attempting to run when failed (#44239) (#44292) + +commit 79c62fd7243e362f4002964a319f519db6a7d6db +Author: Benjamin Trent +Date: Fri Jul 12 15:21:47 2019 -0500 + + [ML][Data Frame] Fixing default delay set in timesync (#44281) (#44293) + + * [ML][Data Frame] Fixing default delay set in timesync + + * disallowing explicit null, don't do duration check on write + +commit d187fcb9de218c634c827f44fdc44947eec198ae +Author: Nikita Glashenko +Date: Fri Jul 12 20:44:59 2019 +0500 + + Support WKT point conversion to geo_point type (#44107) + + This PR adds support for parsing geo_point values from WKT POINT format. + Also, a few minor bugs in geo_point parsing were fixed. + + Closes #41821 + +commit 4e0cc1be8fbac45858b2b2809864d547dacf49cf +Author: Lisa Cawley +Date: Fri Jul 12 09:54:35 2019 -0700 + + [DOCS] Fixes attribute in data frame transform API (#44249) + +commit 3fa677ce79ea08b69103cb7c07de692395ea8e15 +Author: Zachary Tong +Date: Fri Jul 12 12:34:34 2019 -0400 + + Document that pipeline aggs are not compatible with composite agg (#44180) + +commit 7ad9beb087c0bdfd6d347b09b4ac623fbcf2a5eb +Author: James Baiera +Date: Fri Jun 28 11:12:56 2019 -0400 + + Set auto expand replicas on enrich index after force merge is done. (#43600) + +commit 8fdcf28fac25ecd67c93a2aa57da372734f4a4ae +Author: Lisa Cawley +Date: Fri Jul 12 08:26:31 2019 -0700 + + [DOCS] Reformats API parameter details (#44194) + +commit 4d8bf1c3e359336a4148c3238e4c45a07b0a6374 +Author: Lisa Cawley +Date: Fri Jul 12 08:25:23 2019 -0700 + + [DOCS] Removes links to ML tutorial (#44251) + +commit aa6b544facbe4d2eb6f4834723ddd6c9fef28b06 +Author: Lisa Cawley +Date: Fri Jul 12 08:07:12 2019 -0700 + + [DOCS] Moves Watcher troubleshooting page (#44250) + +commit 68cd675892d113d581d320272d4cca0d14aa9de8 +Author: Benjamin Trent +Date: Fri Jul 12 10:10:24 2019 -0500 + + [ML][Data Frame] responding with 409 status code when failing _stop (#44231) (#44276) + + * [ML][Data Frame] responding with appropriate status code when failing _stop + + * adding null checks for persistent task data + + * addressing PR comments + +commit e23ecc58388c57663ee9dbaf66b00bfc572fc755 +Author: Przemyslaw Gomulka +Date: Fri Jul 12 16:53:27 2019 +0200 + + JSON logging refactoring and X-Opaque-ID support backport(#41354) (#44178) + + This is a refactor to current JSON logging to make it more open for extensions + and support for custom ES log messages used inDeprecationLogger IndexingSlowLog , SearchSLowLog + We want to include x-opaque-id in deprecation logs. The easiest way to have this as an additional JSON field instead of part of the message is to create a custom DeprecatedMessage (extends ESLogMEssage) + + These messages are regular log4j messages with a text, but also carry a map of fields which can then populate the log pattern. The logic for this lives in ESJsonLayout and ESMessageFieldConverter. + + Similar approach can be used to refactor IndexingSlowLog and SearchSlowLog JSON logs to contain fields previously only present as escaped JSON string in a message field. + + closes #41350 + backport #41354 + +commit dd5f4ae00eb1f1614ebff1361780559a8394daed +Author: Przemysław Witek +Date: Fri Jul 12 16:49:48 2019 +0200 + + Update .ml-config mappings before indexing job, datafeed or df analytics config (#44216) (#44273) + +commit 9b4f50b40a82f4f9ca8e4a18e30337064dd2abb9 +Author: Armin Braun +Date: Fri Jul 12 15:03:09 2019 +0200 + + Remove Redundant GetAllSnapshots Method from RepositoryData (#44259) (#44271) + + * With the removal of the incompatible snapshots list in RepositoryData + the get snapshots and get all snapshots methods are equivalent so I + removed one of them + +commit 068286ca4b23fdff08e0ac2578a7c0aba0c0c4c8 +Author: Yannick Welsch +Date: Fri Jul 12 14:23:06 2019 +0200 + + Remove RemoteClusterConnection.ConnectedNodes (#44235) + + This instead exposes the set of connected nodes on ConnectionManager. + +commit 40d3c60d7a1f21fde6fd64c4231fe5495cffa9b9 +Author: Przemysław Witek +Date: Fri Jul 12 13:52:26 2019 +0200 + + Make testDatafeedTimingStats_DatafeedJobIdUpdated test easier to debug (#44206) (#44268) + +commit 475752be75253888bfd70c7a57810cca81bb35e6 +Author: Ioannis Kakavas +Date: Fri Jul 12 14:34:15 2019 +0300 + + Make plugin verification FIPS 140 compliant (#44266) + + This change makes the process of verifying the signature of + official plugins FIPS 140 compliant by defaulting to use the + BouncyCastle FIPS provider and adding a dependency to bcpg-fips + that implement parts of openPGP in a FIPS compliant manner. + + In already FIPS 140 enabled environments that use the + BouncyCastle FIPS provider, the bcfips dependency is redundant + but doesn't cause an issue as it will be added only in the classpath + of the cli-tools + + This is a backport of #44224 + +commit 6c02cf0241eca437b34c2e12ea4ccf12d37d0ee0 +Author: Armin Braun +Date: Fri Jul 12 13:18:55 2019 +0200 + + Fix InternalTestCluster StopRandomNode Assertion (#44258) (#44265) + + * The assertion added in #44214 is tripped by tests running dedicated + test clusters per test needlessly.This breaks existing tests like the one in #44245. + * Closes #44245 + +commit ad6dce16f42e2bed35d7e5b754d7a4f9133975ce +Author: Armin Braun +Date: Fri Jul 12 12:45:06 2019 +0200 + + Safer Shard Snapshot Delete (#44165) (#44244) + + * Safer Shard Snapshot Delete + + * We shouldn't delete the snapshot meta file before we update the index + in the shard folder. If we fail to update the index-N after deleting the + existing index-N is broken because the snap- blob it references is gone. + +commit 735c897ec6c10a63e54e78b68d89514e744b657f +Author: David Turner +Date: Fri Jul 12 11:30:52 2019 +0100 + + Avoid counting votes from master-ineligible nodes (#43688) + + Today if a master-eligible node is converted to a master-ineligible node it may + remain in the voting configuration, meaning that the master node may count its + publish responses as an indication that it has properly persisted the cluster + state. However master-ineligible nodes do not properly persist the cluster + state, so it is not safe to count these votes. + + This change adjusts `CoordinationState` to take account of this from a safety + point of view, and also adjusts the `Coordinator` to prevent such nodes from + joining the cluster. Instead, it triggers a reconfiguration to remove from the + voting configuration a node that now appears to be master-ineligible before + processing its join. + + Backport of #43688, see #44260. + +commit 9e920f9612a57a97e633f2704c134b1ca50f42c9 +Author: Armin Braun +Date: Fri Jul 12 12:05:35 2019 +0200 + + Make Timestamps Returned by Snapshot APIs Consistent (#43148) (#44261) + + * We don't have to calculate the start and end times form the shards for the status API, we have the start time available from the CS or the `SnapshotInfo` in the repo and can either take the end time form the `SnapshotInfo` or + take the most recent time from the shard stats for in progress snapshots + * Closes #43074 + +commit e490ecb7d3d5e546cf12bfd0676156fcfffca383 +Author: Albert Zaharovits +Date: Fri Jul 12 11:04:50 2019 +0300 + + Fix X509AuthenticationToken principal (#43932) + + Fixes a bug in the PKI authentication. This manifests when there + are multiple PKI realms configured in the chain, with different + principal parse patterns. There are a few configuration scenarios + where one PKI realm might parse the principal from the Subject + DN (according to the `username_pattern` realm setting) but + another one might do the truststore validation (according to + the truststore.* realm settings). + + This is caused by the two passes through the realm chain, first to + build the authentication token and secondly to authenticate it, and + that the X509AuthenticationToken sets the principal during + construction. + +commit 8d35583c433d5de240dd1d26e7973d83726530e2 +Author: Alpar Torok +Date: Fri Jul 12 10:53:33 2019 +0300 + + Fix port range allocation with large worker IDs (#44213) + + * Fix port range allocation with large worker IDs + + Relates to #43983 + + The IDs gradle uses are incremented for the lifetime of the daemon which + can result in port ranges that are outside the valid range. + This change implements a modulo based formula to wrap the port ranges + when the IDs get too large. + + Adresses #44134 but #44157 is also required to be able to close it. + +commit d2407d0ffcc45bca73a2339a456dee741114d90e +Author: Armin Braun +Date: Fri Jul 12 09:08:17 2019 +0200 + + Remove Redundant Setting of OP_WRITE Interest (#43653) (#44255) + + * Remove Redundant Setting of OP_WRITE Interest + + * We shouldn't have to set OP_WRITE interest before running into a partial write. Since setting OP_WRITE is handled by the `eventHandler.postHandling` logic, I think we can simply remove this operation and simplify/remove tests that were testing the setting of the write interest + +commit 91c342a888d0bb6920f352896cba4312c28fc367 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Jul 12 12:44:01 2019 +1000 + + fix and enable repository-hdfs secure tests (#44044) (#44199) + + Due to recent changes are done for converting `repository-hdfs` to test + clusters (#41252), the `integTestSecure*` tasks did not depend on + `secureHdfsFixture` which when running would fail as the fixture + would not be available. This commit adds the dependency of the fixture + to the task. + + The `secureHdfsFixture` is a `AntFixture` which is spawned a process. + Internally it waits for 30 seconds for the resources to be made available. + For my local machine, it took almost 45 seconds to be available so I have + added the wait time as an input to the `AntFixture` defaults to 30 seconds + and set it to 60 seconds in case of secure hdfs fixture. + + The integ test for secure hdfs was disabled for a long time and so + the changes done in #42090 to fix the tests are also done in this commit. + +commit 263f76e5ea0ecfda9d49d009ec00ae1a711fe9e1 +Author: Mark Vieira +Date: Thu Jul 11 17:13:08 2019 -0700 + + Revert "[DOCS] Moves Watcher troubleshooting page (#44144)" + + This reverts commit 11375926eccc4ad271dc1260e242f3e974a239f5. + +commit 11375926eccc4ad271dc1260e242f3e974a239f5 +Author: Lisa Cawley +Date: Thu Jul 11 14:30:35 2019 -0700 + + [DOCS] Moves Watcher troubleshooting page (#44144) + +commit 9ff8600d46d610250878179d129c1c03c66373c1 +Author: James Rodewig +Date: Thu Jul 11 17:00:45 2019 -0400 + + Revert "[DOCS] Relocate several APIs to REST APIs section (#44238)" + + This reverts commit 6ebd59791afe2e0d55be2989fdbb594972237340. + +commit 3cd96065662a42d4fd594f4824e77fe275c532f6 +Author: Mark Vieira +Date: Thu Jul 11 13:32:23 2019 -0700 + + Mute failing test + +commit 62b5b81fd26f62ba18e2d7bef31f016681778730 +Author: James Rodewig +Date: Thu Jul 11 16:22:57 2019 -0400 + + [DOCS] Relocate several APIs to REST APIs section (#44238) + +commit 0dd06cf7a574b23490b771190d4fe512b362fada +Author: Armin Braun +Date: Thu Jul 11 21:56:36 2019 +0200 + + Remove Dead Code Around Snapshots (#44109) (#44236) + + * Just some random spots that have become unused with recent cleanups + +commit 5698f4dc2704b5c94dc51b46e25bbd3240802969 +Author: Mark Vieira +Date: Thu Jul 11 12:37:31 2019 -0700 + + Pass tests.jvms system property to test tasks for maxParallelForks (#44237) + +commit 7a82106de6c5ce5273e299f78d7d419c742fbc5f +Author: Mark Vieira +Date: Thu Jul 11 12:30:21 2019 -0700 + + Ignore test seed when flag is passed (#44234) + +commit 8030d8f6dcafbfaeaeac9eae9b4b6f217e12f79a +Author: John Murphy +Date: Thu Jul 11 15:07:58 2019 -0400 + + [DOCS] Add `lowercase` filter to phrase suggester example so searches are case insensitive (#44186) + +commit 32cb47b91cef86016bf60d0c957f326e2296fbe5 +Author: Mayya Sharipova +Date: Thu Jul 11 14:14:23 2019 -0400 + + Add l1norm and l2norm distances for vectors (#44116) + + Add L1norm - Manhattan distance + Add L2norm - Euclidean distance + relates to #37947 + +commit b4b2ad3593a4fbff7e6f747ddcfe8c6ff984a208 +Author: Michael Basnight +Date: Thu Jul 11 12:40:09 2019 -0500 + + Ensure enrich policy is immutable (#43604) + + This commit ensures the policy cannot be overwritten. An error is thrown + if the policy exists. All tests have been updated accordingly. + +commit 31725ef39092c49c80ef3abf93abc7631ea8cd1b +Author: Christoph Büscher +Date: Thu Jul 11 20:14:02 2019 +0200 + + [Tests] Increase SimpleQueryStringIT allowed maxClauseCount (#44215) + + For this test, we randomize the CLUSTER_MAX_CLAUSE_COUNT on test setup + (@BeforeClass) between 50 and 100. Some queries in the test generate 56 clauses + which hasn't been an issue before LUCENE-8811, but we slightly need to increase + the minimal possible clause count now. + + Closes #44192 + +commit ae8f625d73343c22ab2f96e5afb5dbd3a664de73 +Author: Yannick Welsch +Date: Thu Jul 11 19:11:42 2019 +0200 + + Report usages old child breakers when breaking on real memory (#44221) + + This will help in investigations where the real memory circuit breaker is tripped to better understand + on what the actual memory is used, i.e. whether it's a temporary thing (e.g. requests) in contrast to + more permanently allocated memory (e.g. accounting). + +commit 40cc081ad3bcac12f98f02e9110738771b6c88c0 +Author: Benjamin Trent +Date: Thu Jul 11 12:50:50 2019 -0500 + + [ML][Data Frame] adds index validations to _start data frame transform (#44191) (#44227) + + * [ML][Data Frame] adds index validations to _start data frame transform + + * addressing pr comments + +commit 2768662822aff1f4d0dfa34b606c975ac8ef3bc7 +Author: Armin Braun +Date: Thu Jul 11 19:35:15 2019 +0200 + + Cleanup Stale Root Level Blobs in Sn. Repository (#43542) (#44226) + + * Cleans up all root level temp., snap-%s.dat, meta-%s.dat blobs that aren't referenced by any snapshot to deal with dangling blobs left behind by delete and snapshot finalization failures + * The scenario that get's us here is a snapshot failing before it was finalized or a delete failing right after it wrote the updated index-(N+1) that doesn't reference a snapshot anymore but then fails to remove that snapshot + * Not deleting other dangling blobs since that don't follow the snap-, meta- or tempfile naming schemes to not accidentally delete blobs not created by the snapshot logic + * Follow up to #42189 + * Same safety logic, get list of all blobs before writing index-N blobs, delete things after index-N blobs was written + +commit e9f9f009407d8eeefbc332bfd5d80c07a49e7701 +Author: Andrei Stefan +Date: Thu Jul 11 20:02:24 2019 +0300 + + SQL: add pretty printing to JSON format (#43756) (#44220) + + (cherry picked from commit cbd9d4c259bf5a541bc49f65f7973174a36df449) + +commit c091b6c004178152ff3a0510e49d134599a0c20d +Author: Christos Soulios +Date: Thu Jul 11 19:20:13 2019 +0300 + + Migrating tests from AvgIT integration test to AvgAggregatorTests (#44076) (#44225) + + This PR migrates most tests from AvgIT integration test to AvgAggregatorTests, as described in #42893 + +commit 2171b6b47fe7a0fa7b555fb80292f3fd436e3bc5 +Author: István Zoltán Szabó +Date: Thu Jul 11 18:05:05 2019 +0200 + + [DOCS] Adds data frame analytics API and evaluate API resource documentation (#43972) + + This PR adds the resource documentation of the data frame analytics APIs and the evaluate API to the ML API doc pool. + +commit 5f22370b6bc08a00ec5cd9821c965446ca632973 +Author: Armin Braun +Date: Thu Jul 11 17:58:00 2019 +0200 + + Fix ShrinkIndexIT (#44214) (#44223) + + * Fix ShrinkIndexIT + + * Move this test suit to cluster scope. Currently, `testShrinkThenSplitWithFailedNode` stops a random node which randomly turns out to be the only shared master node so the cluster reset fails on account of the fact that no shared master node survived. + * Closes #44164 + +commit c82d9c5b50fc5b438455161c33b72b5af67583e9 +Author: Benjamin Trent +Date: Thu Jul 11 09:22:52 2019 -0500 + + [ML] Adds support for regression.mean_squared_error to eval API (#44140) (#44218) + + * [ML] Adds support for regression.mean_squared_error to eval API + + * addressing PR comments + + * fixing tests + +commit 1636701d69e35316cb594b64bacebfeec4a2691d +Author: Igor Motov +Date: Wed Jul 10 16:44:44 2019 -0400 + + CI: Disable SimpleQueryStringIT.testDocWithAllTypes + + Tracked by #44192 + +commit 374030a53fc378521e151188564dc4553546a504 +Author: Nick Knize +Date: Thu Jul 11 09:17:22 2019 -0500 + + Upgrade to lucene-8.2.0-snapshot-860e0be5378 (#44171) (#44184) + + Upgrades lucene library to lucene-8.2.0-snapshot-860e0be5378 + +commit 66a9b721f540f1692ea17de8ba8269d7ac6af910 +Author: Igor Motov +Date: Wed Jul 10 16:53:17 2019 -0400 + + Add Map to XContentParser Wrapper (#44036) + + In some cases we need to parse some XContent that is already parsed into + a map. This is currently happening in handling source in SQL and ingest + processors as well as parsing null_value values in geo mappings. To avoid + re-serializing and parsing the value again or writing another map-based + parser this commit adds an iterator that iterates over a map as if it was + XContent. This makes reusing existing XContent parser on maps possible. + + Relates to #43554 + +commit f01a9eeb348bd14210a0c1ffe79fda88efec18f3 +Author: James Rodewig +Date: Thu Jul 11 09:10:39 2019 -0400 + + [DOCS] Rewrite `has_child` query to use new format (#44190) + +commit 7ba18732f798382ef1a44174f9acb796e0ceedd4 +Author: Alpar Torok +Date: Thu Jul 11 15:04:25 2019 +0300 + + Run some REST tests against a cluster running in docker containers (#39515) + + * Run REST tests against a cluster running on docker + + Closes #38053 + +commit ea5513f2cff601b448fbce807f17a3ea857ec79f +Author: Yannick Welsch +Date: Thu Jul 11 14:02:55 2019 +0200 + + Make NodeConnectionsService non-blocking (#44211) + + With connection management now being non-blocking, we can make NodeConnectionsService + avoid the use of MANAGEMENT threads that are blocked during the connection attempts. + + I had to fiddle a bit with the tests as testPeriodicReconnection was using both the mock Threadpool + from the DeterministicTaskQueue as well as the real ThreadPool initialized at the test class level, + which resulted in races. + +commit 47ab2bda72b67afa2695def007c309e135bf06e7 +Author: Alpar Torok +Date: Thu Jul 11 14:59:05 2019 +0300 + + Improve how log is tailed in testclusters on failure (#40600) + + * Improoce how log is tailed in testclusters on failure + + - only print last few lines + - print all errors and warnings + - compact repeating errors and warnings + +commit eace735d2488f2d36b598a143ebe77b620826b17 +Author: surprisingb <41961041+surprisingb@users.noreply.github.com> +Date: Thu Jul 11 13:57:59 2019 +0200 + + Update discovery-ec2 docs (#43693) + + Fix `discovery.ec2.tag.TAGNAME` example with the correct parameter. + +commit 51f0e941d3dc67be67a8e8b65f1c73a141c62157 +Author: Armin Braun +Date: Thu Jul 11 13:52:36 2019 +0200 + + Reduce Number of List Calls During Snapshot Create and Delete (#44088) (#44209) + + * Reduce Number of List Calls During Snapshot Create and Delete + + Some obvious cleanups I found when investigation the API call count + metering: + * No need to get the latest generation id after loading latest + repository data + * Loading RepositoryData already requires fetching the latest + generation so we can reuse it + * Also, reuse list of all root blobs when fetching latest repo + generation during snapshot delete like we do for shard folders + * Lastly, don't try and load `index--1` (N = -1) repository data, it + doesn't exist -> just return the empty repo data initially + +commit 8ce8c627dd005d586eed938dbecae1a4ea0801bb +Author: Armin Braun +Date: Thu Jul 11 13:52:06 2019 +0200 + + Some Cleanup in o.e.i.shard (#44097) (#44208) + + * Some Cleanup in o.e.i.shard + + * Extract one duplicated method + * Cleanup obviously unused code + +commit 2ee07f1ff4f26608d8adf9f9d164e0b62c451405 +Author: Yannick Welsch +Date: Thu Jul 11 12:11:39 2019 +0200 + + Simplify port usage in transport tests (#44157) + + Simplifies AbstractSimpleTransportTestCase to use JVM-local ports and also adds an assertion so + that cases like #44134 can be more easily debugged. The likely reason for that one is that a test, + which was repeated again and again while always spawning a fresh Gradle worker (due to Gradle + daemon) kept increasing Gradle worker IDs, causing an overflow at some point. + +commit 5886aefeedc0acc792d888227fe58df9b9aa1db8 +Author: David Roberts +Date: Thu Jul 11 11:42:03 2019 +0100 + + [ML] Wait for .ml-config primary before assigning persistent tasks (#44170) + + Now that ML job configs are stored in an index rather than + cluster state, availability of the .ml-config index is very + important to the operation of ML. When a cluster starts up + the ML persistent tasks will be considered for node + assignment very early on. It is best in this case if + assignment is deferred until after the .ml-config index is + available. + + The introduction of data frame analytics jobs has made this + problem worse, because anomaly detection jobs already waited + for the primary shards of the .ml-state, .ml-anomalies-shared + and .ml-meta indices to be available before doing node + assignment, and by coincidence this would probably lead to + the primary shards of .ml-config also being searchable. But + data frame analytics jobs had no other index checks prior to + this change. + + This fixes problem 2 of #44156 + +commit c0ed64bb92fbc3ea8afd67f94eed2e92b2864753 +Author: Armin Braun +Date: Thu Jul 11 11:14:37 2019 +0200 + + Improve Repository Consistency Check in Tests (#44204) + + * Improve Repository Consistency Check in Tests (#44099) + + * Check that index metadata as well as snapshot metadata always exists + when referenced by other metadata + + * Fix SnapshotResiliencyTests on ExtraFS (#44113) + + * As a result of #44099 we're now checking more directories and have to + ignore the `extraN` folders for those like we do for indices already + * Closes #44112 + +commit 8a554f97373ed7d2369c6fb36130ee174b29ffe6 +Author: Armin Braun +Date: Thu Jul 11 07:15:51 2019 +0200 + + Remove IncompatibleSnapshots Logic from Codebase (#44096) (#44183) + + * The incompatible snapshots logic was created to track 1.x snapshots that + became incompatible with 2.x + * It serves no purpose at this point + * It adds an additional GET request to every loading of + RepositoryData (from loading the incompatible snapshots blob) + +commit 4e6cbc28901de746ee25533c8d41a1c6a736b44f +Author: lcawl +Date: Wed Jul 10 17:58:17 2019 -0700 + + [DOCS] Fixes formatting in data frame analytics API + +commit 7c2e4b2857232d6efa84e1f6f6a9884087e6cbba +Author: Mark Vieira +Date: Wed Jul 10 15:42:28 2019 -0700 + + [Backport] Enable caching of rest tests which use integ-test distribution (#44181) + +commit 00b16e332dd9d64785be007eef253453b5347897 +Author: Lisa Cawley +Date: Wed Jul 10 15:12:32 2019 -0700 + + [DOCS] Reformat rollup APIs to use new API format (#44131) + +commit fa36f82277f493f8c57fea8a60c0f9f6c5cce143 +Author: Lisa Cawley +Date: Wed Jul 10 14:39:38 2019 -0700 + + [DOCS] Minor edits to data frame APIs (#44138) + +commit aaf8ba9cb4a69afe728b2a54bef7dc7a94d84a18 +Author: Lisa Cawley +Date: Wed Jul 10 14:38:32 2019 -0700 + + [DOCS] Adds frequency option to data frame transform resource (#44177) + +commit df2e1fb43ea9bf421d815e3643c22a071fc9eeef +Author: Igor Motov +Date: Wed Jul 10 10:20:39 2019 -0400 + + Geo: add validator that only checks altitude (#43893) + + By default, we don't check ranges while indexing geo_shapes. As a + result, it is possible to index geoshapes that contain contain + coordinates outside of -90 +90 and -180 +180 ranges. Such geoshapes + will currently break SQL and ML retrieval mechanism. This commit removes + these restriction from the validator is used in SQL and ML retrieval. + +commit 8fda49a83494b30e554c6e22e49eae51d0a1f6ed +Author: Ryan Ernst +Date: Wed Jul 10 13:42:44 2019 -0700 + + Remove unused import in TransportShardBulkAction + + Accidentally left from backporting #44092 + +commit cbb19032dfb9643056656e82a62cfa8e0a6a0648 +Author: Christoph Büscher +Date: Wed Jul 10 22:41:18 2019 +0200 + + [Test] Additional logging for RemoteClusterClientTests (#44124) + +commit d6f09fdb97bf71cbd6b2b2d260c3658fd98382bb +Author: Armin Braun +Date: Wed Jul 10 22:08:36 2019 +0200 + + Add WARN Logging if Mock Network Accepts Huge Number of Connections (#44169) (#44182) + + * Add WARN Logging if Mock Network Accepts Huge Number of Connections + + * As discussed, added warn logging to rule out endless accept loops for #43387 + * Had to handle it by the relatively awkward override in the mock nio + because we don't have logging in the NIO module where + (`ServerChannelContext` lives) + +commit c6efb9be2a235a7a73475e0a87ff0a273fc3394a +Author: Ryan Ernst +Date: Wed Jul 10 10:54:31 2019 -0700 + + Convert ReplicationResponse to Writeable (#43953) + + This commit convers ReplicationResponse and all its subclasses to + support Writeable.Reader as a constructor. + + relates #34389 + +commit fb77d8f4612bbfc3e025cd85a0757cac2e915cb6 +Author: Ryan Ernst +Date: Tue Jul 9 00:15:39 2019 -0700 + + Removed writeTo from TransportResponse and ActionResponse (#44092) + + The base classes for transport requests and responses currently + implement Streamable and Writeable. The writeTo method on these base + classes is implemented with an empty implementation. Not only does this + complicate subclasses to think they need to call super.writeTo, but it + also can lead to not implementing writeTo when it should have been + implemented, or extendiong one of these classes when not necessary, + since there is nothing to actually implement. + + This commit removes the empty writeTo from these base classes, and fixes + subclasses to not call super and in some cases implement an empty + writeTo themselves. + + relates #34389 + +commit bde5802ad6ebdf982d7aad4cb3d7d49d532d1e4e +Author: Alpar Torok +Date: Tue Jul 9 10:04:03 2019 +0300 + + Test fixtures improovements (#43956) + + * Test fixtures improovements + + Don't disable some of the precommit tasks on fixtures. + This no longer makes sense now that a project can both produce and use a + fixture. + + In order for this to be possible, had to add an additional configuration + to make JarHell class accessible to the task even if it's not a + dependency of the project and fix some of the third party audit fallout + from #43671 which wasn't detected at the time due to the issue being + fixed here. + + Closes #43918 + +commit 5b71340f992e2d6c43e1e5ce5c0449f5fa0f8761 +Author: Lisa Cawley +Date: Wed Jul 10 09:24:33 2019 -0700 + + [DOCS] Moves Watcher limitations (#44141) + +commit d2c3f4bae97c8c3ae10e93d0bdeb79e728b7deb4 +Author: Michael Basnight +Date: Mon Jul 1 12:13:19 2019 -0500 + + Validate read priv of enrich source indices (#43595) + + This commit adds permissions validation on the indices provided in the + enrich policy. These indices should be validated at store time so as not + to have cryptic error messages in the event the user does not have + permissions to access said indices. + +commit 92ad588275fff83d610385f8b52fc8f8285962e4 +Author: Zachary Tong +Date: Wed Jul 10 13:20:28 2019 -0400 + + Remove generic on AggregatorFactory (#43664) (#44079) + + AggregatorFactory was generic over itself, but it doesn't appear we + use this functionality anywhere (e.g. to allow the super class + to declare arguments/return types generically for subclasses to + override). Most places use a wildcard constraint, and even when a + concrete type is specified it wasn't used. + + But since AggFactories are widely used, this led to + the generic touching many pieces of code and making type signatures + fairly complex + +commit b15891954267a50f222e2b6b13e57d0c19af0edc +Author: Nhat Nguyen +Date: Tue Jul 9 09:51:56 2019 -0400 + + Do not use mock engine in PrimaryAllocationIT (#44083) + + PrimaryAllocationIT#testForceStaleReplicaToBePromotedToPrimary + relies on the flushing when a shard is no long assigned. This behavior, + however, can be randomly disabled in MockInternalEngine. + + Closes #44049 + +commit 07f53e39b31359dbadef731281eb24c4b35eaa38 +Author: David Roberts +Date: Wed Jul 10 15:45:57 2019 +0100 + + [ML] Fix ML memory tracker lockup when inner step fails (#44158) + + When the ML memory tracker is refreshed and a refresh is + already in progress the idea is that the second and + subsequent refresh requests receive the same response as + the currently in progress refresh. + + There was a bug that if a refresh failed then the ML + memory tracker's view of whether a refresh was in progress + was not reset, leading to every subsequent request being + registered to receive a response that would never come. + + This change makes the ML memory tracker pass on failures + as well as successes to all interested parties and reset + the list of interested parties so that further refresh + attempts are possible after either a success or failure. + + This fixes problem 1 of #44156 + +commit 4cbd028960126c7990133f4b81e339b486b78d1c +Author: James Rodewig +Date: Wed Jul 10 10:07:49 2019 -0400 + + [DOCS] Correct `ignore_unmapped` parm typo for nested query + +commit bb3e5351b55993105450dca2d213070e78e72cf7 +Author: Andrei Stefan +Date: Wed Jul 10 16:03:44 2019 +0300 + + SQL: double quotes escaping bug fix (#43829) + + (cherry picked from commit d589dcad18c3708913e13c757b91c846aeb35bb4) + +commit 1ae0db7053aad167d8814efb6d7c987520fddb00 +Author: James Rodewig +Date: Wed Jul 10 08:50:54 2019 -0400 + + [DOCS] Rewrite nested query to use new format (#44130) + +commit d0f1a756d995cfe49ce61bf1dfcc198dd5d2af35 +Author: David Turner +Date: Wed Jul 10 13:23:40 2019 +0100 + + Comment on the extra reroute after failing shards (#44152) + + The `ShardFailedClusterStateTaskExecutor` fails some shards, which performs a + reroute, but then sometimes schedules a followup reroute. It's not clear from + the code why this followup is necessary, so this commit adds a short comment + describing why it's necessary. + +commit cad804df926f64ba2d7ea0d2d338cf2c6e1e41f6 +Author: David Roberts +Date: Wed Jul 10 13:21:17 2019 +0100 + + [TEST] Mute ShrinkIndexIT + + Due to https://github.com/elastic/elasticsearch/issues/44164 + +commit 018d946bbaf1a9af033652ca7ac0e9fb73d5d1d7 +Author: Albert Zaharovits +Date: Wed Jul 10 14:05:01 2019 +0300 + + [DOC] Backup & Restore Security Configuration (#42970) + + This commit documents the backup and restore of a cluster's + security configuration. + + It is not possible to only backup (or only restore) security + configuration, independent to the rest of the cluster's conf, + so this describes how a full configuration backup&restore + will include security as well. Moreover, it explains how part + of the security conf data resides on the special .security + index and how to backup that using regular data snapshot API. + + Co-Authored-By: Lisa Cawley + Co-Authored-By: Tim Vernum + +commit 9567f337f5735ee21aa54c76433d764ea6b525e2 +Author: Andrei Stefan +Date: Wed Jul 10 14:10:15 2019 +0300 + + SQL: handle SQL not being available in a more graceful way (#43665) + + * Add test for SQL not being available error message in JDBC. + * Add a new qa sub-project that explicitly disables SQL XPack module in Gradle. + + (cherry picked from commit 8a1ac8a3a88a325ec9b99963e0fa288c18ee0ee5) + +commit 9957b66d2ec4db476640d5930156fce4b344d2bf +Author: Andrei Stefan +Date: Wed Jul 10 14:07:45 2019 +0300 + + SQL: concrete indices array size bug fix (#43878) + + * The created array didn't have the correct initial size while attempting to resolve multiple indices + + (cherry picked from commit 341006e9913e831408f5bbc7f8ad8c453a7f630e) + +commit 44781e415ee1cedf814b68d448085763edffbb69 +Author: Przemysław Witek +Date: Wed Jul 10 11:51:44 2019 +0200 + + [7.x] [ML] Add DatafeedTimingStats to datafeed GetDatafeedStatsAction.Response (#43045) (#44118) + +commit 853ddb5a07175cd5d10ba95f3e5a09f046a30238 +Author: David Roberts +Date: Wed Jul 10 10:26:30 2019 +0100 + + [ML] Fix custom timestamp override with dot-separated fractional seconds (#44127) + + Custom timestamp overrides provided to the find_file_structure + endpoint produced an invalid Grok pattern if the fractional + seconds separator was a dot rather than a comma or colon. + This commit fixes that problem and adds tests for this sort + of timestamp override. + + Fixes #44110 + +commit 913b6a64e8efe1e8a673cbfcf0cb965d0732eb2a +Author: Martijn van Groningen +Date: Wed Jul 10 10:52:22 2019 +0200 + + Replace Streamable w/ Writable for MultiSearchRequest (#44057) + + This commit replaces usages of Streamable with Writeable for the + MultiSearchRequest class. + + I ran into this when developing a custom action that reuses + MultiSearchRequest in the enrich branch. + + Relates to #34389 + +commit cb62d4acdf44d29bc1ceb0148bad8578c7521066 +Author: David Roberts +Date: Wed Jul 10 09:35:23 2019 +0100 + + [ML-DataFrame] Add a frequency option to transform config, default 1m (#44120) + + Previously a data frame transform would check whether the + source index was changed every 10 seconds. Sometimes it + may be desirable for the check to be done less frequently. + This commit increases the default to 60 seconds but also + allows the frequency to be overridden by a setting in the + data frame transform config. + +commit 64ff895a32b13bc3fcbdbb18faadd3336e197e2e +Author: Adrien Grand +Date: Wed Jul 10 09:35:45 2019 +0200 + + Add 7.3 release notes. (#44010) + +commit a23d1ed00dc8e677cdf36b96faee20a74f7c4d80 +Author: Armin Braun +Date: Wed Jul 10 08:12:26 2019 +0200 + + Mute SearchWithRandomExceptionsIT (#44147) (#44149) + + * This is failing quiete often and we can reproduce it now so we don't + need additional test logging on CI + * Relates #40435 + +commit aec44fecbc815959a88deb54b2fef17a84b5ae3f +Author: David Turner +Date: Tue Jul 9 18:43:05 2019 +0100 + + Decouple DiskThresholdMonitor & ClusterInfoService (#44105) + + Today the `ClusterInfoService` requires the `DiskThresholdMonitor` at + construction time so that it can notify it when nodes report changes in their + disk usage, but this is awkward to construct: the `DiskThresholdMonitor` + requires a `RerouteService` which requires an `AllocationService` which comees + from the `ClusterModule` which requires the `ClusterInfoService`. + + Today we break the cycle with a `LazilyInitializedRerouteService` which is + itself a little ugly. This commit replaces this with a more traditional + subject/observer relationship between the `ClusterInfoService` and the + `DiskThresholdMonitor`. + +commit e70cad4c52b504b38fe05da2b9bd2ac622b597fe +Author: David Turner +Date: Tue Jul 9 17:01:48 2019 +0100 + + Remove node conn block after connection barrier (#44114) + + Today `testOnlyBlocksOnConnectionsToNewNodes` fails (extremely rarely) if the + last attempt to connect to `node0` is delayed for so long that the test runs + `nodeConnectionsBlocks.clear()` before the connection attempt obtains the + expected connection block. We can turn this into a reliable failure with this + delay: + + ```diff + diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java + index f48413824d3..9a1d0336bcd 100644 + --- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java + +++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java + @@ -300,6 +300,13 @@ public class NodeConnectionsService extends AbstractLifecycleComponent { + private final Runnable connectActivity = () -> threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() { + @Override + protected void doRun() { + + + + try { + + Thread.sleep(500); + + } catch (InterruptedException e) { + + throw new AssertionError("unexpected", e); + + } + + + assert Thread.holdsLock(mutex) == false : "mutex unexpectedly held"; + transportService.connectToNode(discoveryNode); + consecutiveFailureCount.set(0); + ``` + + This commit reverts the extra logging introduced in #43979 and fixes this + failure by waiting for the connection attempt to hit the barrier before + removing it. + + Fixes #40170 + +commit a406ef1d38a9f6fb749b461bc82917cb4ffabf82 +Author: Mark Vieira +Date: Tue Jul 9 08:55:37 2019 -0700 + + Fix eclipse project file generation (#44080) + +commit 88ee47c9ba3f4df25a3242fe125fb02b98948827 +Author: marcos ramos <42946042+maramos-elastic@users.noreply.github.com> +Date: Tue Jul 9 12:40:33 2019 -0300 + + Fix OIDC documentation settings (#44115) + + Current kibana setting is xpack.security.auth.oidc.realm, + but the correct one is xpack.security.authc.oidc.realm + +commit 23d7e309daa12fbabb0a9f9841967bdfe1a31cb2 +Author: David Kyle +Date: Tue Jul 9 13:10:16 2019 +0100 + + Mute put job docs test + + Relates to #43271 + +commit 389c923a82101d5a46f505458a3e1c82b3b26ac7 +Author: Sachin Frayne <32795683+sachinfrayne@users.noreply.github.com> +Date: Tue Jul 9 12:43:18 2019 +0100 + + [Docs] Fix json syntax in watcher compare condition (#44032) + +commit 268971db0359dc9cc7a336801b723368afabb73e +Author: David Turner +Date: Tue Jul 9 10:59:22 2019 +0100 + + Wait for blackholed connection before discovery (#44077) + + Since #42636 we no longer treat connections specially when simulating a + blackholed connection. This means that at the end of the safety phase we may + have just started a connection attempt which will time out, but the default + timeout is 30 seconds, much longer than the 2 seconds we normally allow for + post-safety-phase discovery. This commit adds time for such a connection + attempt to time out. + + It also fixes some spurious logging of `this` that now refers to an object with + an unhelpful `toString()` implementation introduced in #42636. + + Fixes #44073 + +commit 748a10866dfd0e20970e49bb392f400130288e46 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jul 9 11:49:32 2019 +0200 + + Reindex ScrollableHitSource pump data out (#43864) + + Refactor ScrollableHitSource to pump data out and have a simplified + interface (callers should no longer call startNextScroll, instead they + simply mark that they are done with the previous result, triggering a + new batch of data). This eases making reindex resilient, since we will + sometimes need to rerun search during retries. + + Relates #43187 and #42612 + +commit 859709cc9429d83ca9fa5752c349dae23a29fc2d +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jul 9 11:46:02 2019 +0200 + + Closed index noop recovery during upgrade (#44072) + + Test that closed indices do noop recovery during rolling upgrade. + +commit fd9eebae81a8d6e0d3297dc090b0a080b7975fd4 +Author: David Turner +Date: Tue Jul 9 08:41:42 2019 +0100 + + Only apply initial recovery filter to shrunk shard (#44054) + + Today the `index.routing.allocation.initial_recovery._id` setting can only be + set on indices that are the result of a shrink, but the filtered allocation + decider also applies this filter to shards with a recovery source of + `EMPTY_STORE`. The only way to have this setting set while the recovery source + is `EMPTY_STORE` is to force-allocate an empty primary, but such a forced + allocation ignores this allocation decider. + + This commit simplifies the allocation decider so that the `initial_recovery` + setting only applies to shards with a recovery source of `LOCAL_SHARDS`. + +commit 9eac5ceb1b458c7f79712d9deaf67e04246537d4 +Author: Armin Braun +Date: Tue Jul 9 09:18:25 2019 +0200 + + Dry up inputstream to bytesreference (#43675) (#44094) + + * Dry up Reading InputStream to BytesReference + * Dry up spots where we use the same pattern to get from an InputStream to a BytesReferences + +commit f1ebb82031c681caaa5a41162d62b0e4a655342f +Author: Armin Braun +Date: Tue Jul 9 09:18:03 2019 +0200 + + Update the gcs chunk_size documentation. (#38749) (#44098) + + Remove `1g` from the examples, as the GCS repository chunk_size can be at most 100m. + +commit dc8f8e40eb2fe3f08407e824225d65e0edc02778 +Author: Armin Braun +Date: Tue Jul 9 06:32:08 2019 +0200 + + Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode (#43537) (#44082) + + * Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode + + * See comment in the test: The problem is that when the snapshot delete works out partially on master failover and the retry fails on `SnapshotMissingException` no repository cleanup is run => we still failed even with repo cleanup logic in the delete path now + * Fixed the test by rerunning a create snapshot and delete loop to clean up the repo before verifying file counts + * Closes #39852 + +commit 94578a8b47c9a2ebd1c18ee100b3e0207349e7fe +Author: Lisa Cawley +Date: Mon Jul 8 15:41:45 2019 -0700 + + [DOCS] Defines data frame transform resources (#43996) + + Co-Authored-By: István Zoltán Szabó + +commit 2b1cd58648d0a050b96bc870e1a727d7723b4e73 +Author: Ryan Ernst +Date: Mon Jul 8 17:27:05 2019 -0700 + + Remove ActionResponse uses from HLRC (#44091) + + The rest client does not communicate over the transport protocol. + However, in the move to make all apis supported in the HLRC, some + response classes were copied with extending ActionResponse, which is + meant strictly for the transport protocol. This commit removes uses of + that base class from HLRC. + +commit cd4021274ae1c98a5d0d29a2e1b0155f3d8090ff +Author: lcawl +Date: Mon Jul 8 11:43:18 2019 -0700 + + [DOCS] Enables testing for create job ML API (#44022) + +commit 117f14e0edb8cc58e1b4793396f436ddb8f13fa4 +Author: Lisa Cawley +Date: Mon Jul 8 11:20:57 2019 -0700 + + [DOCS] Updates 7.x version in data frame analytics API (#44026) + +commit efddbcc1d1797da8974d267aa57abc2a49c01107 +Author: Lisa Cawley +Date: Mon Jul 8 10:14:37 2019 -0700 + + [DOCS] Fixes earliest_record_timestamp data type (#44030) + +commit 6dce458ecc26f3fe26d54de484c94fbbfca0d1bd +Author: David Turner +Date: Mon Jul 8 17:27:43 2019 +0100 + + Randomise retention lease expiry time (#44067) + + In today's test suite indices mostly use the default value of `12h` for the + `index.soft_deletes.retention_lease.period` setting, which in the context of + the test suite essentially means "never expires". In fact, the tests should all + behave correctly even if the lease period is much shorter; tests that rely on + leases not expiring should configure their indices appropriately. + + This commit randomises the lease expiry time for those indices created during + tests which do not set a specific value for this setting. + +commit 4b3f1003b073f414104f0959515395abac78f1fd +Author: Lisa Cawley +Date: Mon Jul 8 08:56:30 2019 -0700 + + [DOCS] Reformat freeze unfreeze APis to use new API format (#43948) + +commit 8e8d7667cb89ea810f167339cff8ea2b57369abf +Author: Christoph Büscher +Date: Mon Jul 8 17:33:44 2019 +0200 + + [Tests] Fix type inference issue (#44063) + +commit 03332b5aeb3eeb4c9fc0b61e08a8b84c33d09ee8 +Author: Armin Braun +Date: Mon Jul 8 17:21:40 2019 +0200 + + Don't Consistency Check Broken Repository in Test (#43499) (#44071) + + * Missed this one in #42189 and it randomly runs into a situation where the broken mock repo is broken such that we can't get to a consistent end state via a delete + * Closes #43498 + +commit 251287f89d2b7cda0b9d95aa37c12138f3b94daf +Author: Tanguy Leroux +Date: Mon Jul 8 17:06:21 2019 +0200 + + Check again on-going snapshots/restores of indices before closing (#43873) + + Today we prevent any index that is actively snapshotted or restored to be closed. + This verification is done during the execution of the first phase of index closing + (ie before blocking the indices). + + We should also do this verification again in the last phase of index closing + (ie after the shard sanity checks and right before actually changing the index + state and the routing table) because a snapshot/restore could sneak in while + the shards are verified-before-close. + +commit 0c8294e633c1353c47ac0f93145c02544c373920 +Author: Alpar Torok +Date: Mon Jul 8 17:49:33 2019 +0300 + + Make sure the clean task doesn't break test fixtures (#43641) + + Use a dedicated fixture dir. + +commit 299a52c17d3f0bd4d619ed0421f5aa92b43639df +Author: Mark Tozzi +Date: Mon Jul 8 10:46:23 2019 -0400 + + Enable validating user-supplied missing values on unmapped fields (#43718) (#43940) + + Provides a hook for aggregations to introspect the `ValuesSourceType` for a user supplied Missing value on an unmapped field, when the type would otherwise be `ANY`. Mapped field behavior is unchanged, and still applies the `ValuesSourceType` of the field. This PR just provides the hook for doing this, no existing aggregations have their behavior changed. + +commit 4390d4a8af7da5dd4173d64dc92dbca4f8c6eeaa +Author: James Rodewig +Date: Mon Jul 8 08:56:51 2019 -0400 + + [DOCS] Clarify array is not a field datatype (#43931) + +commit 2918363e90188290140f69022155ba6fcc2b5c01 +Author: Armin Braun +Date: Mon Jul 8 14:57:27 2019 +0200 + + Simplify BlobStoreRepository (Flatten Nested Classes) (#42833) (#44060) + + * In the current codebase it is hardly obvious what code operates on a shard and is run by a datanode what code operates on the global metadata and is run on master + * Fixed by adjusting the method names accordingly + * The nested context classes don't add much if any value, they simply spread out the parameters that go into a shard snapshot create or delete all over the place since their + constructors can be inlined in all spots + * Fixed by flattening the nested classes into BlobStoreRepository + * Also: + * Inlined the other single use inner classes + +commit afe81fd62580278d31b89594364d39c78a7ecd54 +Author: Armin Braun +Date: Mon Jul 8 14:15:31 2019 +0200 + + Some Cleanup in Test Framework (#44039) (#44059) + + * Remove some obvious dead code + * Move assert methods that were only used in a single test class to the child they belong to + * Inline some redundant methods + +commit 5fc12917c358976cb7cfb97d1b794797d95b696c +Author: David Kyle +Date: Mon Jul 8 11:52:19 2019 +0100 + + Data frame task failure does not make a 500 response (#44058) + + Data frame task responses had logic to return a HTTP 500 status code if there was + any node or task failures even if other tasks in the same request reported correctly. + This is different to how other task responses are handled where a 200 is always + returned leaving the client should check for failures. Returning a 500 also breaks + the high level rest client so always return a 200 + + Closes #44011 + +commit 3f3bcb23c2d27452a2e7e3d14b1b8f34353152ac +Author: David Turner +Date: Mon Jul 8 06:22:36 2019 +0100 + + AwaitsFix testForceStaleReplicaToBePromotedToPrimary + + Relates #44049 + +commit 3129f5b42ed490e0d285295e2d6c18b33d132759 +Author: David Turner +Date: Mon Jul 8 10:31:15 2019 +0100 + + Do not copy initial recovery filter during split (#44053) + + If an index is the result of a shrink then it will have a value set for + `index.routing.allocation.initial_recovery._id`. If this index is subsequently + split then this value will be copied over, forcing the initial allocation of + the split shards to occur on the node on which the shrink took place. Moreover + if this node no longer exists then the split will fail. This commit suppresses + the copying of this setting when splitting an index. + + Fixes #43955 + +commit af9b98e81c001ba9589823d1a84d90b556ee8022 +Author: Armin Braun +Date: Mon Jul 8 10:55:39 2019 +0200 + + Recursively Delete Unreferenced Index Directories (#42189) (#44051) + + * Use ability to list child "folders" in the blob store to implement recursive delete on all stale index folders when cleaning up instead of using the diff between two `RepositoryData` instances to cover aborted deletes + * Runs after ever delete operation + * Relates #13159 (fixing most of this issues caused by unreferenced indices, leaving some meta files to be cleaned up only) + +commit 247f2dabad2103534ddd0b6df4d1c37e44d604ee +Author: Przemyslaw Gomulka +Date: Mon Jul 8 09:56:01 2019 +0200 + + Fix decimal point parsing for date_optional_time backport(#43859) #44050 + + Joda allowed for date_optional_time and strict_date_optional_time a decimal point to be . dot or , comma + For our java.time implementation we should also extend this for strict_date_optional_time-nanos + the approach to fix this is the same as in iso8601 parser + closes #43730 + +commit 2176d09c373861e2e8f83b46d2a9521621391faa +Author: Armin Braun +Date: Mon Jul 8 08:10:01 2019 +0200 + + Provide an Option to Use Path-Style-Access with S3 Repo (#41966) (#44046) + + * Provide an Option to Use Path-Style-Access with S3 Repo + + * As discussed, added the option to use path style access back again and + deprecated it. + * Defaulted to `false` + * Added warning to docs + + * Closes #41816 + +commit 9beb51fc448abada3ce52bd0424541e9c60bcd1d +Author: Ioannis Kakavas +Date: Mon Jul 8 08:52:21 2019 +0300 + + Revert "Mute testEnableDisableBehaviour (#42929)" + + This reverts commit 6ee578c6eb3f2a6164934720ee82e2456d7bb226. + +commit f6efc55556a8acf3f3c83aa2b824b34665e6ffc0 +Author: Armin Braun +Date: Sun Jul 7 19:59:16 2019 +0200 + + Fix SnapshotResiliencyTest (#44015) (#44041) + + * Closes #43989 + +commit 990ac4ca838f6221b25bec9c894af39eca8971eb +Author: Armin Braun +Date: Sun Jul 7 19:50:46 2019 +0200 + + Some Cleanup in BlobStoreRepository (#43323) (#44043) + + * Some Cleanup in BlobStoreRepository + + * Extracted from #42833: + * Dry up index and shard path handling + * Shorten XContent handling + +commit 9089820d8f4b04b3d58bcacd1e83e73f26ce435e +Author: Nhat Nguyen +Date: Fri Jul 5 18:55:24 2019 -0400 + + Enable indexing optimization using sequence numbers on replicas (#43616) + + This PR enables the indexing optimization using sequence numbers on + replicas. With this optimization, indexing on replicas should be faster + and use less memory as it can forgo the version lookup when possible. + This change also deactivates the append-only optimization on replicas. + + Relates #34099 + +commit d3ddedf9fc3a1807c2737b2887aacaade7822e84 +Author: Dimitris Athanasiou +Date: Sat Jul 6 02:03:29 2019 +0300 + + [7.x][ML] Add missing doc links to df-analytics rest spec and HLRC javadocs (#44025) (#44033) + +commit 37e1ad7062ac3c1814e98c0997fba2672a7321dd +Author: Mayya Sharipova +Date: Fri Jul 5 17:59:13 2019 -0400 + + Forbid empty doc values on vector functions (#43944) + + Currently when a document misses a vector value, vector function + returns 0 as a score for this document. We think this is incorrect + behaviour. + With this change, an error will be thrown if vector functions are + used with docs that are missing vector doc values. + Also VectorScriptDocValues is modified to allow size() function, + which can be used to check if a document has a value for the + vector field. + +commit a1a62fded31e2e03c24acdf813376ab104bdebf2 +Author: Dimitris Athanasiou +Date: Fri Jul 5 23:10:45 2019 +0300 + + [7.x][ML] Stop df-analytics action request should filter tasks (#44016) (#44023) + + As a `BaseTasksRequest`, `StopDataFrameAnalyticsAction.Request` should + implement a `match` method that makes sure only df-analytics tasks + are applied. + +commit 504a43d43ad47d1af2bf39cc9f941c5911804b15 +Author: Yannick Welsch +Date: Fri Jul 5 19:35:06 2019 +0200 + + Move ConnectionManager to async APIs (#42636) + + This commit converts the ConnectionManager's openConnection and connectToNode methods to + async-style. This will allow us to not block threads anymore when opening connections. This PR also + adapts the cluster coordination subsystem to make use of the new async APIs, allowing to remove + some hacks in the test infrastructure that had to account for the previous synchronous nature of the + connection APIs. + +commit 8bfe18477ec41202e953e0dd6954f2b95e6fc1c3 +Author: Nhat Nguyen +Date: Fri Jul 5 13:55:25 2019 -0400 + + Clarify consequence of translog async setting (#44020) + + Relates #43915 + +commit a831d4707c805f4794db58c1a8b0b601bbb52e8c +Author: lcawl +Date: Fri Jul 5 10:56:09 2019 -0700 + + [DOCS] Temporarily disables data frame API testing + +commit 88783927d1feb0276213f82486531658ddf1200b +Author: Yannick Welsch +Date: Fri Jul 5 18:27:17 2019 +0200 + + Weaken assertion in PublicationTransportHandler (#44014) + + These assertions do not hold true when a master fails during publication and quickly becomes + master again, publishing a new cluster state in a higher term which races against the previous + cluster state publication to self (which does not matter anyway). + + Relates #43994 + + Closes #44012 + +commit 5aeb7368015dcd7a7044b7dc67cf7e34f6f15030 +Merge: 7242267f5de 01b982fd313 +Author: István Zoltán Szabó +Date: Fri Jul 5 14:26:47 2019 +0200 + + Merge branch '7.x' of github.com:elastic/elasticsearch into 7.x + +commit 7242267f5de3c23c6b804c24c1536af6a2327974 +Author: István Zoltán Szabó +Date: Fri Jul 5 13:34:05 2019 +0200 + + [DOCS] Adds data frame analytics APIs to the ML APIs (#43875) + + This PR adds the reference documentation pages of the data frame analytics APIs (PUT, START, STOP, GET, GET stats, DELETE, Evaluate) to the ML APIs pool. + +commit 01b982fd313802b75817571733340020ea29aa19 +Author: Akshesh Doshi +Date: Fri Jul 5 18:42:56 2019 +0700 + + Draw attention to transport layer in remote cluster docs (#43883) + + Closes #43858 + +commit 1220ff5b6da1edb9064451660b5a81258f22b529 +Author: Yannick Welsch +Date: Fri Jul 5 12:53:20 2019 +0200 + + Publish to self through transport (#43994) + + This commit ensures that cluster state publications to self also go through the transport layer. This + allows voting-only nodes to intercept the publication to self. + + Fixes an issue discovered by a test failure where a voting-only node, which was the only + bootstrapped node, would not step down as master after state transfer because publishing to self + would succeed. + + Closes #43631 + +commit 5cdf3ff3faad5db19d62bc82f32f6655f394390d +Author: Yannick Welsch +Date: Fri Jul 5 11:02:42 2019 +0200 + + Revert "[TEST] Mute RemoteClusterServiceTests.testCollectNodes" + + This reverts commit d8a2970fa40d34676242d520502fcd00a2a2fbfa. + +commit d090fa514f46584a33a4071ece72711024f762ac +Author: Yannick Welsch +Date: Fri Jul 5 10:54:12 2019 +0200 + + Use unique ports per test worker (#43983) + + * Use unique ports per test worker + + * Add test for system property + + * check presence of tests.gradle + + * Revert "check presence of tests.gradle" + + This reverts commit 2fee7512a28f95c94c5bf7a3312e808f918a9510. + +commit 06df0c0a4c452fca3c30be04ebad95a71968d1e0 +Author: David Turner +Date: Fri Jul 5 09:52:38 2019 +0100 + + Improve RetentionLease(Bgrd)SyncAction#toString() (#43987) + + Today `RetentionLeaseSyncAction.Request` and + `RetentionLeaseBackgroundSyncAction.Request` both describe themselves as + `Request{...}` in the value returned from their respective `toString()` + methods. This commit adds the name of the owning class to both so we have + something a bit easier to search for and so we can distinguish foreground from + background syncs in logs and test failures and so on. + +commit 435a83f3fdfe5f65b02f04b477d5ba10b5436034 +Author: David Turner +Date: Thu Jul 4 16:32:46 2019 +0100 + + Add more logging to testOnlyBlocksOnConnectionsToNewNodes (#43979) + + Some more output from this occasionally-failing test tracked in #40170. + +commit 4c3e71b61ac0787f666331679340e0f87e1e6ee3 +Author: István Zoltán Szabó +Date: Fri Jul 5 09:38:21 2019 +0200 + + [DOCS] Adds description to the preview data frame transform API (#43745) + +commit 30b20920b9c3419f8b9155a81b0623a8275bc143 +Author: Dimitris Athanasiou +Date: Fri Jul 5 10:28:57 2019 +0300 + + [7.x][ML] Report correct count for df-analytics get-stats API (#43969) (#43981) + + The count should match the number of all df-analytics that + matched the id in the request. However, we set the count + to the number of df-analytics returned which was bound to the + `size` parameter. This commit fixes this by setting the count + to the count of the `get` response. + +commit adc06ffd89543b0ec9c4d0786a0a15cfb3c5d273 +Author: Martijn van Groningen +Date: Fri Jul 5 08:05:58 2019 +0200 + + take builtin role into account in docs tests + +commit cdf55cb5c55664453e224b1c566d090fc4a0df4d +Author: Jim Ferenczi +Date: Thu Jul 4 22:48:48 2019 +0200 + + Refactor index engines to manage readers instead of searchers (#43860) + + This commit changes the way we manage refreshes in the index engines. + Instead of relying on a SearcherManager, this change uses a ReaderManager that + creates ElasticsearchDirectoryReader when needed. Searchers are now created on-demand + (when acquireSearcher is called) from the current ElasticsearchDirectoryReader. + It also slightly changes the Engine.Searcher to extend IndexSearcher in order + to simplify the usage in the consumer. + +commit 4128b9b4f795b0fe30a2b74ab32311607a8c10d7 +Author: Hendrik Muhs +Date: Thu Jul 4 21:39:24 2019 +0200 + + audit message missing for autostop + + call onStop when auto stopping (#43984) + + fixes #43977 + +commit 688bf1b3885c1bc419043e7f9d86e0bd87ad1cb6 +Author: lcawl +Date: Thu Jul 4 09:13:00 2019 -0700 + + [DOCS] Fixes broken link + +commit 9528c59fb39748b18cc862ae765403f2a0432e0a +Author: Martijn van Groningen +Date: Thu Jul 4 17:37:48 2019 +0200 + + added a basic test that enriching data works + +commit a030e3e513ed2d7ddd47bcad175cd3c954e38296 +Author: Lisa Cawley +Date: Thu Jul 4 08:22:17 2019 -0700 + + [DOCS] Reformat CCR APIs to use new API format (#43952) + +commit aeb3c1fd1b0d25da5696441155aa25961a3bf5a9 +Author: Christoph Büscher +Date: Thu Jul 4 16:40:31 2019 +0200 + + Prevent types deprecation warning for indices.exists requests (#43963) + + Currently we log a deprecation warning to the types removal in + RestGetIndicesAction even if the REST method is HEAD, which is used by the + indices.exists API. Since the body is empty in this case we should not need to + show the deprecation warning. + + Closes #43905 + +commit b037aeaa6e842309369e41e27843990d67416430 +Author: Tanguy Leroux +Date: Thu Jul 4 17:06:37 2019 +0200 + + Fix IndexShardIT.testIndexCanChangeCustomDataPath() (#43978) + + The test IndexShardIT.testIndexCanChangeCustomDataPath() fails + on 7.x and 7.3 because the translog cannot be recovered. + + While I can't reproduce the issue, I think it has been introduced in #43752 + which changed ReadOnlyEngine so that it opens the translog in its + constructor in order to load the translog stats. This opening writes a + new checkpoint file, but because 7.x/7.3 does not wait for shards to be + started after being closed, the test immediately starts to copy shard files + to a new directory and possibly does not copy all the required translog files. + + By waiting for the shards to be started after being closed, we ensure + that the shards (and engines) have been correctly initialized and that + the translog checkpoint file is not currently being written. + + closes #43964 + +commit 1dd3d14f09f64ab2822ff54fe6c664ea0c75c619 +Author: Martijn van Groningen +Date: Thu Jul 4 16:40:33 2019 +0200 + + take into account `manage_enrich` builtin role + +commit ac119b07e7d206ac82f67615700166f062e6bd6a +Merge: 7ba6e1752a7 36f72597376 +Author: Martijn van Groningen +Date: Thu Jul 4 15:50:11 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 36f725973760fd51ba8a171770f58872020dbc20 +Author: Benjamin Trent +Date: Thu Jul 4 07:28:47 2019 -0500 + + [ML] Fix datafeed checks when a concrete remote index is present (#43923) + + A bug was introduced in 6.6.0 when we added support for + rollup indices. Rollup caps does NOT support looking at + remote indices, consequently, since we always look up rollup + caps, the datafeed fails with an error if its config + includes a concrete remote index. (When all remote indices + in a datafeed config are wildcards the problem did not + occur.) + + The rollups feature does not support remote indices, so if + there is any remote index in a datafeed config (wildcarded + or not), we can skip the rollup cap checks. This PR + implements that change. + +commit 2a70df424d8bc14acbd6d1ed5e1c960f4863eda2 +Author: Dimitris Athanasiou +Date: Thu Jul 4 15:17:36 2019 +0300 + + [TEST][ML] Fix assertion after starting df-analytics job (#43957) (#43967) + + In MachineLearningIT.testStopDataFrameAnalytics we call start and + then assert the state is `started`. However, if things go fast enough, + the state could have already changed to `reindexing` or `analyzing`. + The test has been failing occasionally due to the state being + `reindexing`. We fix this by simply asserting the state is either + of `started`, `reindexing` or `analyzing`. + + Closes #43924 + +commit 7ba6e1752a72bc3e5af0b2c132b61f0dfc26292c +Author: Martijn van Groningen +Date: Thu Jul 4 13:17:22 2019 +0200 + + required changes after merge + +commit 653f1436a067a2273d50a4593ba246d35bbb9605 +Merge: 397150fa1e7 4b99255fedd +Author: Martijn van Groningen +Date: Thu Jul 4 13:05:10 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 4b99255fedd25cf4bc213b74cb097bd938c755ef +Author: Alan Woodward +Date: Thu Jul 4 11:23:27 2019 +0100 + + Add name() method to TokenizerFactory (#43909) + + This brings TokenizerFactory into line with CharFilterFactory and TokenFilterFactory, + and removes the need to pass around tokenizer names when building custom analyzers. + + As this means that TokenizerFactory is no longer a functional interface, the commit also + adds a factory method to TokenizerFactory to make construction simpler. + +commit 1b6109517a86d011e9c8c1b5023df67b6c250f00 +Author: Alpar Torok +Date: Thu Jul 4 12:12:14 2019 +0300 + + Mute failing test + + Tracking in #43960 + +commit 2cc0a56fe698f907efe95c79b5e0d6e59caef66a +Author: Jim Ferenczi +Date: Thu Jul 4 09:38:56 2019 +0200 + + Fix wrong logic in `match_phrase` query with multi-word synonyms (#43941) + + Disjunction over two individual terms in a phrase query with multi-word synonyms + wrongly applies a prefix query to each of these terms. This change fixes this bug + by inversing the logic to use prefixes on `phrase_prefix` queries only. + + Closes #43308 + +commit cacc3f7ff85383dbf32c8d0f6a8653be6246645d +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jul 1 09:27:55 2019 +0200 + + Async IO Processor release before notify (#43682) + + This commit changes async IO processor to release the promiseSemaphore + before notifying consumers. This ensures that a bad consumer that + sometimes does blocking (or otherwise slow) operations does not halt the + processor. This should slightly increase the concurrency for shard + fsync, but primarily improves safety so that one bad piece of code has + less effect on overall system performance. + +commit c5930851043522c69613f68b2ac3b8ac3dfbcfbe +Author: Igor Motov +Date: Wed Jul 3 10:50:32 2019 -0400 + + Geo: Refactors libs/geo parser to provide serialization logic as well (#43717) + + Enables libs/geo parser to return a geometry format object that can + perform both serialization and deserialization functions. This can + be useful for ingest nodes that are trying to modify an existing + geometry in the source. + + Relates to #43554 + +commit 7063a404115153d044dca2b3529e783f6da21900 +Author: Benjamin Trent +Date: Wed Jul 3 16:39:38 2019 -0500 + + [7.x] [ML][Data Frame] Adding bwc tests for pivot transform (#43506) (#43929) + + * [ML][Data Frame] Adding bwc tests for pivot transform (#43506) + + * [ML][Data Frame] Adding bwc tests for pivot transform + + * adding continuous transforms + + * adding continuous dataframes to bwc + + * adding continuous data frame tests + + * Adding rolling upgrade tests for continuous df + + * Fixing test + + * Adjusting indices used in BWC, and handling NPE for seq_no_stats + + * updating and muting specific bwc test + + * Adjusting bwc tests for backport + +commit 553f783e735b24a5eea3730d3ce97c7af74bfcc2 +Author: Przemyslaw Gomulka +Date: Wed Jul 3 20:39:48 2019 +0200 + + Fix DieWithDignity test when waiting on jps backport(#43861) (#43871) + + the test often hangs on executing jps command + we don't need to wait for this command to finish. + + closes #43413 + +commit 680edbe3f10989ce9972e6086062a555cebde112 +Author: Adrien Grand +Date: Wed Jul 3 20:32:04 2019 +0200 + + Bump current version to 7.4. (#43927) + +commit 50e96f9f0e9a967fb5e368c05287ea8d0e8e1316 +Author: Lisa Cawley +Date: Wed Jul 3 11:09:34 2019 -0700 + + [DOCS] Updates documentation version (#43937) + +commit be20fb80e444151754875515e84d9bced83803f7 +Author: Armin Braun +Date: Wed Jul 3 17:14:57 2019 +0200 + + Recursive Delete on BlobContainer (#43281) (#43920) + + This is a prerequisite of #42189: + + * Add directory delete method to blob container specific to each implementation: + * Some notes on the implementations: + * AWS + GCS: We can simply exploit the fact that both AWS and GCS return blobs lexicographically ordered which allows us to simply delete in the same order that we receive the blobs from the listing request. For AWS this simply required listing without the delimiter setting (so we get a deep listing) and for GCS the same behavior is achieved by not using the directory mode on the listing invocation. The nice thing about this is, that even for very large numbers of blobs the memory requirements are now capped nicely since we go page by page when deleting. + * For Azure I extended the parallelization to the listing calls as well and made it work recursively. I verified that this works with thread count `1` since we only block once in the initial thread and then fan out to a "graph" of child listeners that never block. + * HDFS and FS are trivial since we have directory delete methods available for them + * Enhances third party tests to ensure the new functionality works (I manually ran them for all cloud providers) + +commit 49d69bf98792dccdb4c4408555952b0cef2c2135 +Author: Alan Woodward +Date: Wed Jul 3 15:45:35 2019 +0100 + + Actually close IndexAnalyzers contents (#43914) + + IndexAnalyzers has a close() method that should iterate through all its wrapped + analyzers and close each one in turn. However, instead of delegating to the + analyzers' close() methods, it instead wraps them in a Closeable interface, + which just returns a list of the analyzers. In addition, whitespace normalizers are + ignored entirely. + +commit 3250cc53f0ec62eaa80edeb7955bb2f2b430d781 +Author: Alpar Torok +Date: Wed Jul 3 17:31:48 2019 +0300 + + Mute failing test + + Tracked in #43924 + +commit 397150fa1e73954d750f2a0dde22bb1f0f3ddc72 +Author: Martijn van Groningen +Date: Wed Jul 3 15:40:57 2019 +0200 + + Add enrich coordinator proxy action (#43801) + + Introduced proxy api the handle the search request load that originates + from enrich processor. The enrich processor can execute many search + requests that execute asynchronously in parallel and that can easily overwhelm + the search thread pool on nodes. In order to protect this the Coordinator + queues the search requests and only executes a fixed number of search requests + in parallel. + + Besides this; the Coordinator tries to include as much as possible search requests + (up to a defined maximum) inside a multi search request in order to reduce the number + of remote api calls to be made from the node that performs ingestion. + +commit f8fd4321f8c08bf7117491c711229c728f091e90 +Author: Zachary Tong +Date: Tue Jul 2 13:10:46 2019 -0400 + + Link rare_terms docs from index page (#43882) + + Docs for rare_terms were added in #35718, but neglected to + link it from the bucket index page + +commit 9cecc31cdca604e8451c4a3f419a59227807eda8 +Author: David Turner +Date: Wed Jul 3 14:14:03 2019 +0100 + + Shortcut simple patterns ending in `*` (#43904) + + When profiling a call to `AllocationService#reroute()` in a large cluster + containing allocation filters of the form `node-name-*` I observed a nontrivial + amount of time spent in `Regex#simpleMatch` due to these allocation filters. + Patterns ending in a wildcard are not uncommon, and this change treats them as + a special case in `Regex#simpleMatch` in order to shave a bit of time off this + calculation. It also uses `String#regionMatches()` to avoid an allocation in + the case that the pattern's only wildcard is at the start. + + Microbenchmark results before this change: + + Result "org.elasticsearch.common.regex.RegexStartsWithBenchmark.performSimpleMatch": + 1113.839 ±(99.9%) 6.338 ns/op [Average] + (min, avg, max) = (1102.388, 1113.839, 1135.783), stdev = 9.486 + CI (99.9%): [1107.502, 1120.177] (assumes normal distribution) + + Microbenchmark results with this change applied: + + Result "org.elasticsearch.common.regex.RegexStartsWithBenchmark.performSimpleMatch": + 433.190 ±(99.9%) 0.644 ns/op [Average] + (min, avg, max) = (431.518, 433.190, 435.456), stdev = 0.964 + CI (99.9%): [432.546, 433.833] (assumes normal distribution) + + The microbenchmark in question was: + + @Fork(3) + @Warmup(iterations = 10) + @Measurement(iterations = 10) + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @State(Scope.Benchmark) + @SuppressWarnings("unused") //invoked by benchmarking framework + public class RegexStartsWithBenchmark { + + private static final String testString = "abcdefghijklmnopqrstuvwxyz"; + private static final String[] patterns; + + static { + patterns = new String[testString.length() + 1]; + for (int i = 0; i <= testString.length(); i++) { + patterns[i] = testString.substring(0, i) + "*"; + } + } + + @Benchmark + public void performSimpleMatch() { + for (int i = 0; i < patterns.length; i++) { + Regex.simpleMatch(patterns[i], testString); + } + } + } + +commit 3317169c4f88bac8fc546322600821141ca311c2 +Author: Armin Braun +Date: Wed Jul 3 15:09:28 2019 +0200 + + Fix GCS Blob Repository 3rd Party Tests (#43030) (#43913) + + * We have to strip the trailing slash from child names here like we do for AWS + * closes #43029 + +commit e2a9a787fc70008423af2bbfe67394cc94381e22 +Author: James Rodewig +Date: Wed Jul 3 08:55:50 2019 -0400 + + [DOCS] Rewrite dis max query (#43586) + +commit cff027499a56c2a3d0370888f5fc4c3e6e624f6b +Author: paulward24 <52216289+paulward24@users.noreply.github.com> +Date: Wed Jul 3 04:37:07 2019 -0700 + + Ensure to access RecoveryState#fileDetails under lock + + Closes #43840 + +commit 7059224668cf42032e795170f7f8012195f7b421 +Author: Armin Braun +Date: Wed Jul 3 13:26:35 2019 +0200 + + Optimize Snapshot Finalization (#42723) (#43908) + + * Optimize Snapshot Finalization + + * Delete index-N blobs and segement blobs in one single bulk delete instead of in separate ones to save RPC calls on implementations that have bulk deletes implemented + * Don't fail snapshot because deleting old index-N failed, this results in needlessly logging finalization failures and makes analysis of failures harder going forward as well as incorrect index.latest blobs + +commit 662f517f4e56a3b261d36f104229284eb5ae9ba8 +Author: Christoph Büscher +Date: Wed Jul 3 11:27:37 2019 +0200 + + Add _reload_search_analyzers endpoint to HLRC (#43733) + + This change adds the new endpoint that allows reloading of search analyzers to + the high-level java rest client. + + Relates to #43313 + +commit 96b0b27f18ac6fab914d1b75120230df6c82d8c7 +Author: Dimitris Athanasiou +Date: Wed Jul 3 12:41:56 2019 +0300 + + [7.x][ML] Set df-analytics task state to failed when appropriate (#43880) (#43906) + + This introduces a `failed` state to which the data frame analytics + persistent task is set to when something unexpected fails. It could + be the process crashing, the results processor hitting some error, + etc. The failure message is then captured and set on the task state. + From there, it becomes available via the _stats API as `failure_reason`. + + The df-analytics stop API now has a `force` boolean parameter. This allows + the user to call it for a failed task in order to reset it to `stopped` after + we have ensured the failure has been communicated to the user. + + This commit also adds the analytics version in the persistent task + params as this allows us to prevent tasks to run on unsuitable nodes in + the future. + +commit 1e0f67fb384d40c43d95daae7f032ffa20774492 +Author: Jay Modi +Date: Wed Jul 3 03:31:55 2019 -0600 + + Deprecate transport profile security type setting (#43237) + + This commit deprecates the `transport.profiles.*.xpack.security.type` + setting. This setting is used to configure a profile that would only + allow client actions. With the upcoming removal of the transport client + the setting should also be deprecated so that it may be removed in + a future version. + +commit 455b12a4fbabf449a5461f487f071eb3b5bbfef1 +Author: Armin Braun +Date: Wed Jul 3 11:30:49 2019 +0200 + + Add Ability to List Child Containers to BlobContainer (#42653) (#43903) + + * Add Ability to List Child Containers to BlobContainer (#42653) + + * Add Ability to List Child Containers to BlobContainer + * This is a prerequisite of #42189 + +commit 9077c4402f8549690e821430adb438baa2139eb0 +Author: Alexander Reelsen +Date: Wed Jul 3 11:04:57 2019 +0200 + + Watcher: Allow to execute actions for each element in array (#41997) + + This adds the ability to execute an action for each element that occurs + in an array, for example you could sent a dedicated slack action for + each search hit returned from a search. + + There is also a limit for the number of actions executed, which is + hardcoded to 100 right now, to prevent having watches run forever. + + The watch history logs each action result and the total number of actions + the were executed. + + Relates #34546 + +commit 2a8f30eb9a62481917370e4e89877c53d801739a +Author: Tim Vernum +Date: Wed Jul 3 19:08:28 2019 +1000 + + Support builtin privileges in get privileges API (#43901) + + Adds a new "/_security/privilege/_builtin" endpoint so that builtin + index and cluster privileges can be retrieved via the Rest API + + Backport of: #42134 + +commit deacc2038eb24a1f39de7efa52ae1bd4eeda2346 +Author: Tim Vernum +Date: Wed Jul 3 19:07:16 2019 +1000 + + Always attach system user to internal actions (#43902) + + All valid licenses permit security, and the only license state where + we don't support security is when there is a missing license. + However, for safety we should attach the system (or xpack/security) + user to internally originated actions even if the license is missing + (or, more strictly, doesn't support security). + + This allows all nodes to communicate and send internal actions (shard + state, handshake/pings, etc) even if a license is transitioning + between a broken state and a valid state. + + Relates: #42215 + Backport of: #43468 + +commit cd2972239cf2f31d2b605ea73a07511d013d17f2 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Sun Jun 30 17:43:04 2019 +0200 + + AsyncIOProcessor preserve thread context (#43729) + + AsyncIOProcessor now preserves thread context, ensuring that deprecation + warnings are not duplicated to other concurrent operations on the same + shard. + +commit 31b19bd022ce8f17f1f5b0c82e8d3c83e0b05d1e +Author: Tim Vernum +Date: Wed Jul 3 18:04:06 2019 +1000 + + Use separate BitSet cache in Doc Level Security (#43899) + + Document level security was depending on the shared + "BitsetFilterCache" which (by design) never expires its entries. + + However, when using DLS queries - particularly templated ones - the + number (and memory usage) of generated bitsets can be significant. + + This change introduces a new cache specifically for BitSets used in + DLS queries, that has memory usage constraints and access time expiry. + + The whole cache is automatically cleared if the role cache is cleared. + Individual bitsets are cleared when the corresponding lucene index + reader is closed. + + The cache defaults to 50MB, and entries expire if unused for 7 days. + + Backport of: #43669 + +commit 05c0cff1b6ac0eddc81323335551af54b05337a6 +Author: Jim Ferenczi +Date: Wed Jul 3 09:21:11 2019 +0200 + + Fix index_prefix sub field name on nested text fields (#43862) + + This change fixes the name of the index_prefix sub field when the `index_prefix` + option is set on a text field that is nested under an object or a multi-field. + We don't use the full path of the parent field to set the index_prefix field name + so the field is registered under the wrong name. This doesn't break queries since + we always retrieve the prefix field through its parent field but this breaks other + APIs like _field_caps which tries to find the parent of the `index_prefix` field + in the mapping but fails. + + Closes #43741 + +commit 826f38cd706ef9f67fcb4a067e9f5ee29558a14a +Author: Armin Braun +Date: Wed Jul 3 09:28:39 2019 +0200 + + Enable Parallel Deletes in Azure Repository (#42783) (#43886) + + * Parallel deletes via private thread pool + +commit 365dfe88caf33b88dfa24181732533a572c4382b +Author: Tanguy Leroux +Date: Wed Jul 3 08:39:04 2019 +0200 + + Refresh translog stats after translog trimming in NoOpEngine (#43825) + + This commit changes NoOpEngine so that it refreshes its translog + stats once translog is trimmed. + + Relates #43156 + +commit 461aa39daf182af7eb68236195de2b5dd971b6e3 +Author: Tim Vernum +Date: Wed Jul 3 16:29:28 2019 +1000 + + Switch WriteActionsTests.testBulk to use hamcrest (#43897) + + If an item in the bulk request fails, that could be for a variety of + reasons - it may be that the underlying behaviour of security has + changed, or it may just be a transient failure during testing. + + Simply asserting a `true`/`false` value produces failure messages that + are difficult to diagnose and debug. Using hamcert (`assertThat`) will + make it easier to understand the causes of failures in this test. + + Backport of: #43725 + +commit 14884c871f3ec12cb04a4ff954a4cefa983ad271 +Author: Tim Vernum +Date: Wed Jul 3 13:51:44 2019 +1000 + + Document API-Key APIs require manage_api_key priv (#43869) + + Add the "Authorization" section to the API key API docs. + These APIs require The new manage_api_key cluster privilege. + + Relates: #43865 + Backport of: #43811 + +commit 6e9ccda2c59605a8aafe43185d67ccd40142715f +Author: Jake Landis +Date: Tue Jul 2 21:33:30 2019 -0500 + + ilm test - allow more time for policy completion (#43844) + +commit 0a79f4ca70beffe02890d617aecf611ff4213040 +Author: Jake Landis +Date: Tue Jul 2 21:32:05 2019 -0500 + + Extend timeout for TimeSeriesLifecycleActionsIT> testFullPolicy (#43891) + +commit 2dc056b0a0b2fda13710c76b99b6d7a07eb4e018 +Author: Jake Landis +Date: Tue Jul 2 20:44:33 2019 -0500 + + Read the default pipeline for bulk upsert through an alias (#41963) (#42802) + + This commit allows bulk upserts to correctly read the default pipeline + for the concrete index that belongs to an alias. + + Bulk upserts are modeled differently from normal index requests such that + the index request is a request inside of the update request. The update + request (outer) contains the index or alias name is not part of the (inner) + index request. This commit adds a secondary check against the update request + (outer) if the index request (inner) does not find an alias. + +commit a4e518b640240228d894eab1322a9aee0b98435f +Author: Deb Adair +Date: Mon Jul 1 15:34:32 2019 -0700 + + [DOCS] Revise GS intro and remove redundant conceptual content. Closes #43846. + +commit 756c42f99fe6a4a385c64a8469bada617f654ff2 +Author: Mayya Sharipova +Date: Tue Jul 2 21:14:16 2019 -0400 + + Add dims parameter to dense_vector mapping (#43444) (#43895) + + Typically, dense vectors of both documents and queries must have the same + number of dimensions. Different number of dimensions among documents + or query vector indicate an error. This PR enforces that all vectors + for the same field have the same number of dimensions. It also enforces + that query vectors have the same number of dimensions. + +commit fb825a64705fdcc73b2b3fbc8ba775bcb5e597d0 +Author: Benjamin Trent +Date: Tue Jul 2 19:35:37 2019 -0500 + + [7.x] [ML][Data Frame] add node attr to GET _stats (#43842) (#43894) + + * [ML][Data Frame] add node attr to GET _stats (#43842) + + * [ML][Data Frame] add node attr to GET _stats + + * addressing testing issues with node.attributes + + * adjusting for backport + +commit 2c97e26ce8ab904a1dff4d1ee4acd433ab0d331e +Author: Benjamin Trent +Date: Tue Jul 2 19:35:09 2019 -0500 + + [ML][Data Frame] fix progress measurement for continuous transforms (#43838) (#43887) + + * [ML][Data Frame] fix progress measurement for continuous transforms + + * Update DataFrameIndexer.java + +commit 8755448a1858f3e6f85616df42cab07ed8c347b9 +Author: Jack Conradson +Date: Tue Jul 2 15:38:16 2019 -0700 + + Add Datetime Now to Painless Documentation (#43852) + + This change explains why Painless doesn't natively support datetime now, and + gives examples of how to create a version of now through user-defined + parameters. + +commit eb73bed40d1495ff855685b9052ad94ee5c930e7 +Author: Jake Landis +Date: Tue Jul 2 12:16:06 2019 -0500 + + 7x watcher backport testfixes (#43848) + + * fix org.elasticsearch.xpack.watcher.test.integration.RejectedExecutionTests (#41777) + + This commit un-mutes org.elasticsearch.xpack.watcher.test.integration.RejectedExecutionTests + which was failing intermittently due to a logic bug. It is not possible to use the real + Watcher scheduler (which is needed for this test) and reliabliby count the .triggered-watches + since current count of documents in the .triggered-watches index is based on the timing of the + scheduler and the ability to delete based on the Watcher and Write thread pools. + + This commit simply removes the .triggered-watch check and relies soley on the .watcher-history + index as an indication that operations that can occur when the Watcher threadpool is rejecting. + + closes #41734 + + * fix unlikely bug that can prevent Watcher from restarting (#42030) + + The bug fixed here is unlikely to happen. It requires ES to be started with + ILM disabled, Watcher enabled, and Watcher explicitly stopped and restarted. + Due to template validation Watcher does not fully start and can result in a + partially started state. This is an unlikely scenerio outside of the testing + framework. + + Note - this bug was introduced while the test that would have caught it was + muted. The test remains muted since the underlying cuase of the random failures + has not been identified. When this test is un-muted it will now work. + +commit 8e44f5d8456ecd03dc7bc1dc2d3370c8b755e79a +Author: David Roberts +Date: Tue Jul 2 17:48:43 2019 +0100 + + [ML-Data Frame] Add data frame transform cluster privileges to HLRC (#43879) + + Adds the monitor_data_frame_transforms and + manage_data_frame_transforms cluster privileges to + the high level rest client. + + The ALL_ARRAY variable is only used in randomized + tests at the within the Elasticsearch code, so it's + not a major problem that these cluster privileges + weren't added from the start. But since ALL_ARRAY + is public HLRC users may be using it to find out + which cluster privileges exist, so it's best that + it contains them all. + +commit 31cf96e7bfe0806341dc2012ee1768086f1ae1f6 +Author: Christoph Büscher +Date: Tue Jul 2 18:08:04 2019 +0200 + + Return reloaded analyzers in _reload_search_ananlyzer response (#43813) + + Currently the repsonse of the "_reload_search_analyzer" endpoint contains the + index names and nodeIds of indices were analyzers reloading was triggered. This + change add the names of the search-time analyzers that were reloaded. + + Closes #43804 + +commit cc7c5ab2c0dba9918964e16dd5752e6c3db19c31 +Author: Yannick Welsch +Date: Tue Jul 2 18:48:29 2019 +0200 + + Clarify voting-only master node docs (#43857) + + Clarifies the roles of a dedicated voting-only master-eligible node. + + Co-Authored-By: James Rodewig + Co-Authored-By: David Turner + +commit 697cd494bff1d1c5ba694d4c49535a29152de83c +Author: Nhat Nguyen +Date: Tue Jul 2 10:53:25 2019 -0400 + + Remove sort by primary term when reading soft-deletes (#43845) + + With Lucene rollback (#33473), we should never have more than one + primary term for each sequence number. Therefore we don't have to sort + by the primary term when reading soft-deletes. + +commit 1ea53979b55193777f77eefdc12bc28db6ebd396 +Author: Dimitris Athanasiou +Date: Tue Jul 2 16:00:54 2019 +0300 + + [7.x][ML] Get df-analytics action should require monitor privilege (#43831) (#43866) + +commit 8d099dad382109c20d9748273f44111881d4a82d +Author: Tim Vernum +Date: Tue Jul 2 21:57:42 2019 +1000 + + Add "manage_api_key" cluster privilege (#43865) + + This adds a new cluster privilege for manage_api_key. Users with this + privilege are able to create new API keys (as a child of their own + user identity) and may also get and invalidate any/all API keys + (including those owned by other users). + + Backport of: #43728 + +commit b95ee7ebb22939812122871023c3560de250d890 +Author: Benjamin Trent +Date: Tue Jul 2 06:52:34 2019 -0500 + + [7.x] [ML][Data Frame] using transform creation version for node assignment (#43764) (#43843) + + * [ML][Data Frame] using transform creation version for node assignment (#43764) + + * [ML][Data Frame] using transform creation version for node assignment + + * removing unused imports + + * Addressing PR comment + + * adjusing for backport + +commit 82c1ddc11751932e1f08521391c1217ed7784aab +Author: Benjamin Trent +Date: Tue Jul 2 06:52:14 2019 -0500 + + [7.x] [ML][Data Frame] Add deduced mappings to _preview response payload (#43742) (#43849) + + * [ML][Data Frame] Add deduced mappings to _preview response payload (#43742) + + * [ML][Data Frame] Add deduced mappings to _preview response payload + + * updating preview docs + + * fixing code for backport + +commit b977f019b868fdd13e65f3e7a1a0c084d718a55c +Author: Tanguy Leroux +Date: Tue Jul 2 13:39:00 2019 +0200 + + Expose translog stats in ReadOnlyEngine (#43752) (#43823) + + Backport of #43752 for 7.x. + +commit c8ed2719378c11afa2b5d822740952904e99e216 +Author: Ioannis Kakavas +Date: Tue Jul 2 14:20:29 2019 +0300 + + Use URLEncoder#encode(String, String) + + as URLEncoder#encode(String, Charset) is only available since Java + 10 + +commit 4ea17b76dc6b774106230773cf755c3b10594bb0 +Author: Ioannis Kakavas +Date: Tue Jul 2 13:34:38 2019 +0300 + + Fix credentials encoding for OIDC token request (#43808) + + As defined in https://tools.ietf.org/html/rfc6749#section-2.3.1 + both client id and client secret need to be encoded with the + application/x-www-form-urlencoded encoding algorithm when used as + credentials for HTTP Basic Authentication in requests to the OP. + + Resolves #43709 + +commit 4cdb24bceb3689184eeb7f8939b10e1ba50edbdd +Author: Tomas Della Vedova +Date: Tue Jul 2 11:06:29 2019 +0200 + + Use explicit string keys in data_frame test (#43854) + +commit ac7e1476a0e064ea608086e6c80579853f1f70fd +Author: Alexander Reelsen +Date: Tue Jul 2 09:06:14 2019 +0200 + + Update docs to refer to 6.8 instead of 6.7 (#43685) + + A few places in the documentation had mentioned 6.7 as the version to + upgrade from, when doing an upgrade to 7.0. While this is technically + possible, this commit will replace all those mentions to 6.8, as this is + the latest version with the latest bugfixes, deprecation checks and + ugprade assistant features - which should be the one used for upgrades. + + Co-Authored-By: James Rodewig + +commit 1e8e85797d9d807e703c88fe9ad0d36846964e21 +Author: David Turner +Date: Tue Jul 2 06:37:33 2019 +0100 + + Rename and refactor RoutingService (#43827) + + The `RoutingService` has a confusing name, since it doesn't really have + anything to do with routing. Its responsibility is submitting reroute commands + to the master. + + This commit renames this class to `BatchedRerouteService`, and extracts the + `RerouteService` interface to avoid passing `BiConsumer`s everywhere. It also + removes that `BatchedRerouteService extends AbstractLifecycleComponent` since + this service has no meaningful lifecycle. Finally, it introduces a small + wrapper class to allow for lazy initialization to deal with the dependency loop + when constructing a `Node`. + +commit 81f60652d8c4763b3470d148239c96bcd0c29553 +Author: Jack Conradson +Date: Mon Jul 1 21:29:01 2019 -0700 + + Add timezone documentation for Painless datetimes (#43768) + +commit 4eb89a691269ca75c7f9466f2075e336a55dd4f8 +Author: Albert Zaharovits +Date: Tue Jul 2 00:10:15 2019 +0300 + + UserRoleMapper non-null groups and metadata (#43836) + + This is an odd backport of #41774 + + UserRoleMapper.UserData is constructed by each realm and it is used to + "match" role mapping expressions that eventually supply the role names + of the principal. + + This PR filters out `null` collection values (lists and maps), for the groups + and metadata, which get to take part in the role mapping, in preparation + for using Java 9 collection APIs. It filters them as soon as possible, during + the construction. + +commit b5d47e564358cadd115013d1d954728d1d5517b7 +Author: Mark Vieira +Date: Mon Jul 1 11:54:58 2019 -0700 + + Upgrade to Gradle 5.5 (#43788) (#43832) + +commit fe3f9f0c6bca4be3e88370e652bff83e5daee687 +Author: Christoph Büscher +Date: Mon Jul 1 16:25:40 2019 +0200 + + Yet another `the the` cleanup (#43815) + +commit 785aedebade8e4c0736f7a5f65373e0c449691d9 +Author: Martijn van Groningen +Date: Mon Jul 1 17:29:38 2019 +0200 + + Add restart node enrich tests. (#43579) + + This test verifies that enrich policies still exist after a full + cluster restart. If EnrichPolicy is not registered as named xcontent + in EnrichPlugin class then this test fails. + +commit eb0c7196cb2b5c096c2bbaed0f602e1b2bf2f16f +Author: Alpar Torok +Date: Mon Jul 1 18:13:26 2019 +0300 + + Disable Gradle running in parallel + +commit 031d5e96ac2d6e3a12a3d3b03192a391d8b3e8b4 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jul 2 00:55:02 2019 +1000 + + HLRC changes for kerberos grant type (#43642) (#43822) + + The TODO from last PR for kerbero grant type was missed. + This commit adds the changes for kerberos grant type in HLRC. + +commit 1e47ea5f187315cb3757b564c44dd81e8c5bfed5 +Author: Zachary Tong +Date: Mon Jul 1 10:50:57 2019 -0400 + + Update rare_term version skips, fix SetBackedScalingCuckooFilter javadoc + +commit ea1794832f5fd39a2549232db7aaa0aeaa22abbe +Author: Zachary Tong +Date: Mon Jul 1 10:02:36 2019 -0400 + + Add RareTerms aggregation (#35718) + + This adds a `rare_terms` aggregation. It is an aggregation designed + to identify the long-tail of keywords, e.g. terms that are "rare" or + have low doc counts. + + This aggregation is designed to be more memory efficient than the + alternative, which is setting a terms aggregation to size: LONG_MAX + (or worse, ordering a terms agg by count ascending, which has + unbounded error). + + This aggregation works by maintaining a map of terms that have + been seen. A counter associated with each value is incremented + when we see the term again. If the counter surpasses a predefined + threshold, the term is removed from the map and inserted into a cuckoo + filter. If a future term is found in the cuckoo filter we assume it + was previously removed from the map and is "common". + + The map keys are the "rare" terms after collection is done. + +commit 8108834534b39ca8f5f3a95c56874a5e031c1fe5 +Author: Benjamin Trent +Date: Mon Jul 1 09:14:44 2019 -0500 + + [ML][Data Frame] account for delay in writing stats docs (#43703) (#43819) + +commit 4c95c0c456a6e4c406d33ecd8f1577a6ceb13456 +Author: Benjamin Trent +Date: Mon Jul 1 09:14:26 2019 -0500 + + [ML][Data Frame] reduce audit frequency, change log msg, and level (#43771) (#43818) + +commit 755442058140bc064c6f6be059e39d1392b20a79 +Author: Tanguy Leroux +Date: Mon Jul 1 15:19:44 2019 +0200 + + Update docs for Open/Close API (#43809) + + Relates #43530 + +commit 598e00a689056fcbdf72963a1d501248b610f08c +Author: Nhat Nguyen +Date: Sun Jun 30 17:47:38 2019 -0400 + + Make peer recovery send file info step async (#43792) + + Relates #36195 + +commit 9aa6f7c43449b0f8c67339b2da357f49c439012e +Author: Dimitrios Liappis +Date: Mon Jul 1 15:33:34 2019 +0300 + + Update TLS configuration in Docker docs (#43816) + + Following the removal of the `unzip` package from the Elasticsearch + Docker image in #39040, update setup instructions for TLS in Docker. + + Also avoid cross-platform ownership+permission issues by not relying + on local bind mounts for storing generated certs and don't require + `curl` locally installed. + + Backport of #43748 + +commit 13887c01ccc314480ab8e2f9849ffc38ce0def51 +Author: Mark Vieira +Date: Wed Jun 26 14:24:36 2019 -0700 + + Remove compile-time dependency on test fixtures (#43651) + +commit 40d43e3f87ad06375ecaa1718062f7ceaf329e96 +Author: David Turner +Date: Mon Jul 1 12:39:12 2019 +0100 + + Avoid IP addresses for bootstrapping in setup docs (#43802) + + Removes the suggestion to use IP addresses for `cluster.initial_master_nodes` + in the "important settings" discovery docs, leaving only the suggestion to use + node names. + + Relates #41179, #41569 + +commit ffa5919d7caf829ae7ac9fc0a634c0b50f7e7f59 +Author: Julie Tibshirani +Date: Mon Jul 1 12:08:50 2019 +0300 + + Add support for 'flattened object' fields. (#43762) + + This commit merges the `object-fields` feature branch. The new 'flattened + object' field type allows an entire JSON object to be indexed into a field, and + provides limited search functionality over the field's contents. + +commit b4303290d97778ada1fe5b563e6d73b650b69ee0 +Author: Alpar Torok +Date: Mon Jul 1 11:29:54 2019 +0300 + + Testclusters: improove timeout handling (#43440) + +commit a58d231f4d1663589b5518821534f7fbb1894af4 +Author: Hendrik Muhs +Date: Mon Jul 1 10:29:08 2019 +0200 + + relax trigger count for transform stats test (#43753) + + relax trigger count test as we can not guarantee it due to async behaviour + +commit 717d14a7e2666d56dd168addf553a83cc89ff0b5 +Author: Alpar Torok +Date: Mon Jul 1 10:38:56 2019 +0300 + + Backport: convert x pack qa (#43763) + + * Revert "Revert "Test clusters: convert x-pack qa tests (#43283)" (#43549)" + + This reverts commit 8d9a971259bdbcec0a68894c64bea61675f05ada. + + * Fix failing test + +commit 8f3387e7cb680f963a42e5f40adbe5fa9d4510ce +Author: Martijn van Groningen +Date: Mon Jul 1 08:31:31 2019 +0200 + + fixed compile errors after cherry-picking + +commit 3bdb9d5f08e82fa00c7f0d713e96f032246b73c5 +Author: Dimitris Athanasiou +Date: Mon Jul 1 09:19:04 2019 +0300 + + [7.x][ML] Correct df-analytics version introduced to 7.3.0 (#43784) (#43795) + +commit 28358fdbedbc217743d480318fd294b3e63ef556 +Author: weizijun +Date: Mon Jul 1 14:04:26 2019 +0800 + + show a full ingest example in the index page, to let user fast understand ingest node. (#43476) + +commit 237f2bd60a40c7f4d64d094ec523bf51908ecf30 +Author: Martijn van Groningen +Date: Mon Jul 1 07:59:50 2019 +0200 + + Make ingest executing non blocking (#43361) + + Added an additional method to the Processor interface to allow a + processor implementation to make a non blocking call. + + Also added semaphore in order to avoid search thread pools from rejecting + search requests originating from the match processor. This is a temporary workaround. + +commit 3a2c698ce0c7f86a61503caa468fa670facfe540 +Author: Ryan Ernst +Date: Sun Jun 30 22:00:17 2019 -0700 + + Rename Action to ActionType (#43778) + + Action is a class that encapsulates meta information about an action + that allows it to be called remotely, specifically the action name and + response type. With recent refactoring, the action class can now be + constructed as a static constant, instead of needing to create a + subclass. This makes the old pattern of creating a singleton INSTANCE + both misnamed and lacking a common placement. + + This commit renames Action to ActionType, thus allowing the old INSTANCE + naming pattern to be TYPE on the transport action itself. ActionType + also conveys that this class is also not the action itself, although + this change does not rename any concrete classes as those will be + removed organically as they are converted to TYPE constants. + + relates #34389 + +commit adcba69d96a9d68f72ecd69b0327454499c4fdbf +Author: Martijn van Groningen +Date: Sun Jun 30 21:40:40 2019 +0200 + + required changes after merge + +commit eb8e03bc8bbc78bd9004ffdd901535becbe80714 +Merge: 683e116601b 8f49d011135 +Author: Martijn van Groningen +Date: Sun Jun 30 21:32:51 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 8f49d01113500b1f8e405fbe2142c0bb85547bed +Author: Dimitris Athanasiou +Date: Sun Jun 30 19:37:00 2019 +0300 + + [7.x][ML] Rename df-analytics `_id_copy` to `ml__id_copy` (#43754) (#43783) + + Renames `_id_copy` to `ml__id_copy` as field names starting with + underscore are deprecated. The new field name `ml__id_copy` was + chosen as an obscure enough field that users won't have in their data. + Otherwise, this field is only intented to be used by df-analytics. + +commit fca7a1971302b5541272f3a8f350dac3c079b779 +Author: David Turner +Date: Sun Jun 30 16:44:57 2019 +0100 + + Avoid parallel reroutes in DiskThresholdMonitor (#43381) + + Today the `DiskThresholdMonitor` limits the frequency with which it submits + reroute tasks, but it might still submit these tasks faster than the master can + process them if, for instance, each reroute takes over 60 seconds. This causes + a problem since the reroute task runs with priority `IMMEDIATE` and is always + scheduled when there is a node over the high watermark, so this can starve any + other pending tasks on the master. + + This change avoids further updates from the monitor while its last task(s) are + still in progress, and it measures the time of each update from the completion + time of the reroute task rather than its start time, to allow a larger window + for other tasks to run. + + It also now makes use of the `RoutingService` to submit the reroute task, in + order to batch this task with any other pending reroutes. It enhances the + `RoutingService` to notify its listeners on completion. + + Fixes #40174 + Relates #42559 + +commit 55b3ec8d7b5020770be9ee02545933313c174680 +Author: Nhat Nguyen +Date: Sat Jun 29 18:26:08 2019 -0400 + + Make peer recovery clean files step async (#43787) + + Relates #36195 + +commit 5e17bc5dcc52b1c3dc92e51a6f154547aef3d24a +Author: Albert Zaharovits +Date: Sat Jun 29 23:26:17 2019 +0300 + + Consistent Secure Settings #40416 + + Introduces a new `ConsistentSecureSettingsValidatorService` service that exposes + a single public method, namely `allSecureSettingsConsistent`. The method returns + `true` if the local node's secure settings (inside the keystore) are equal to the + master's, and `false` otherwise. Technically, the local node has to have exactly + the same secure settings - setting names should not be missing or in surplus - + for all `SecureSetting` instances that are flagged with the newly introduced + `Property.Consistent`. It is worth highlighting that the `allSecureSettingsConsistent` + is not a consensus view across the cluster, but rather the local node's perspective + in relation to the master. + +commit b599c68d23fe4acc20d50142b8fc03d5b00c49cf +Author: David Roberts +Date: Sat Jun 29 07:51:29 2019 +0100 + + [ML] Assert that a no-op job creates no results nor state (#43681) + + If a job is opened and then closed and does nothing in + between then it should not persist any results or state + documents. This change adapts the no-op job test to + assert no results in addition to no state, and to log + any documents that cause this assertion to fail. + + Relates elastic/ml-cpp#512 + Relates #43680 + +commit 28ab77a023cfea977ba83ae089a4061f7bb34621 +Author: Ryan Ernst +Date: Fri Jun 28 21:40:00 2019 -0700 + + Add StreamableResponseAction to aid in deprecation of Streamable (#43770) + + The Action base class currently works for both Streamable and Writeable + response types. This commit intorduces StreamableResponseAction, for + which only the legacy Action implementions which provide newResponse() + will extend. This eliminates the need for overriding newResponse() with + an UnsupportedOperationException. + + relates #34389 + +commit 7951c63b919d97a1a64e8f8abd7f64a8a14c49c9 +Author: David Roberts +Date: Fri Jun 28 20:06:22 2019 +0100 + + [ML] Mark ml-cpp dependency as regularly changing (#43760) + + Since #41817 was merged the ml-cpp zip file for any + given version has been cached indefinitely by Gradle. + This is problematic, particularly in the case of the + master branch where the version 8.0.0-SNAPSHOT will + be in use for more than a year. + + This change tells Gradle that the ml-cpp zip file is + a "changing" dependency, and to check whether it has + changed every two hours. Two hours is a compromise + between checking on every build and annoying developers + with slow internet connections and checking rarely + causing bug fixes in the ml-cpp code to take a long + time to propagate through to elasticsearch PRs that + rely on them. + +commit 9d5c66be41eea80683f23c5523f09134dbc09a78 +Author: Martijn van Groningen +Date: Fri Jun 28 21:14:59 2019 +0200 + + Migrate watcher hlrc response tests to use AbstractResponseTestCase (#43478) + + Relates to #43472 + +commit d8fe0f5c13b97b1b24dcd6bedf6a04a44e380a33 +Author: James Rodewig +Date: Fri Jun 28 12:56:22 2019 -0400 + + [DOCS] Rewrite `terms_set` query (#43060) + +commit 67a3c656c35c4861b662bc52ac845a8c365cd978 +Author: Benjamin Trent +Date: Fri Jun 28 10:02:37 2019 -0500 + + [7.x] [ML][Data Frame] removing format support (#43659) (#43747) + + * [ML][Data Frame] removing format support (#43659) + + * Fixing conflicts + +commit f02cbe9e402d0f96c001ed80b6a10bdcc863e3f7 +Author: Tanguy Leroux +Date: Fri Jun 28 16:18:54 2019 +0200 + + Trim translog for closed indices (#43156) + + Today when an index is closed all its shards are forced flushed + but the translog files are left around. As explained in #42445 + we'd like to trim the translog for closed indices in order to + consume less disk space. This commit reuses the existing + AsyncTrimTranslogTask task and reenables it for closed indices. + + At the time the task is executed, we should have the guarantee + that nothing holds the translog files that are going to be removed. + It also leaves a short period of time (10 min) during which translog + files of a recently closed index are still present on disk. This could + also help in some cases where the closed index is reopened + shortly after being closed (in order to update an index setting + for example). + + Relates to #42445 + +commit 7ca69db83fd9a3677307ceac098f78340ed40825 +Author: Jim Ferenczi +Date: Fri Jun 28 15:42:43 2019 +0200 + + Refactor IndexSearcherWrapper to disallow the wrapping of IndexSearcher (#43645) + + This change removes the ability to wrap an IndexSearcher in plugins. The IndexSearcherWrapper is replaced by an IndexReaderWrapper and allows to wrap the DirectoryReader only. This simplifies the creation of the context IndexSearcher that is used on a per request basis. This change also moves the optimization that was implemented in the security index searcher wrapper to the ContextIndexSearcher that now checks the live docs to determine how the search should be executed. If the underlying live docs is a sparse bit set the searcher will compute the intersection + betweeen the query and the live docs instead of checking the live docs on every document that match the query. + +commit 377c4cfdc043a7f54eafe744501c92d01ab045c6 +Author: weizijun +Date: Fri Jun 28 21:47:24 2019 +0800 + + Fix threshold spelling errors (#43326) + + Substitutes treshold by threshold + +commit 23f739b513710878b1e95e1693d4b0a1d565aa06 +Author: Alpar Torok +Date: Fri Jun 28 16:38:17 2019 +0300 + + Testclusters: Convert additional projects (#43625) + + * Testclusters: Convert additional projects + + Found some more that were not using testclusters from elasticsearch-ci/1 + + * Allow IOException too + + * Make the client more resilient + +commit d1a4d8866d55ada6ef64d619a10d66eb6636769f +Author: Alpar Torok +Date: Fri Jun 28 16:33:24 2019 +0300 + + Add missing dependencies so we can build in parallel (#43672) + +commit 81dbcfb26815c7012b4d36dcdc20f843603ec11f +Author: Alan Woodward +Date: Fri Jun 28 13:58:06 2019 +0100 + + Wildcard intervals (#43691) + + This commit adds a wildcard intervals source, similar to the prefix. It + also changes the term parameter in prefix to read prefix, to bring it + in to line with the pattern parameter in wildcard. + + Closes #43198 + +commit 74dd6e49fcd9ee83e7ab3b60b68db4aea65737a0 +Author: James Rodewig +Date: Fri Jun 28 08:35:20 2019 -0400 + + [DOCS] Rewrite boosting query (#43647) + +commit 632da7f2c86ca51fba10dd97aabf29ab46a2a832 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jun 28 12:58:22 2019 +0200 + + Enabled cannot be updated (#43701) + + Removed the invalid tip that enabled can be updated for existing fields + and clarified instead that it cannot. + + Related to #33566 and #33933 + +commit 86c853a7c2277c513227de3babfb03a13b2956c4 +Author: Dimitris Athanasiou +Date: Fri Jun 28 13:28:25 2019 +0300 + + [7.x][ML] Rename outlier score setting to feature_influence_threshold (#43705) (#43734) + + Renames outlier score setting `minimum_score_to_write_feature_influence` + to `feature_influence_threshold`. + +commit cab879118d429544c1fc53b976313792317b5944 +Author: Dimitris Athanasiou +Date: Fri Jun 28 13:28:03 2019 +0300 + + [7.x][ML] Support multiple source indices for df-analytics (#43702) (#43731) + + This commit adds support for multiple source indices. + In order to deal with multiple indices having different mappings, + it attempts a best-effort approach to merge the mappings assuming + there are no conflicts. In case conflicts exists an error will be + returned. + + To allow users creating custom mappings for special use cases, + the destination index is now allowed to exist before the analytics + job runs. In addition, settings are no longer copied except for + the `index.number_of_shards` and `index.number_of_replicas`. + +commit 2cc7f5a744844addb22336fe2e6e3642ae422e4f +Author: Christoph Büscher +Date: Fri Jun 28 09:55:40 2019 +0200 + + Allow reloading of search time analyzers (#43313) + + Currently changing resources (like dictionaries, synonym files etc...) of search + time analyzers is only possible by closing an index, changing the underlying + resource (e.g. synonym files) and then re-opening the index for the change to + take effect. + + This PR adds a new API endpoint that allows triggering reloading of certain + analysis resources (currently token filters) that will then pick up changes in + underlying file resources. To achieve this we introduce a new type of custom + analyzer (ReloadableCustomAnalyzer) that uses a ReuseStrategy that allows + swapping out analysis components. Custom analyzers that contain filters that are + markes as "updateable" will automatically choose this implementation. This PR + also adds this capability to `synonym` token filters for use in search time + analyzers. + + Relates to #29051 + +commit 51b230f6ab7d9dee95e6413ddf2c0d0eb2c0e560 +Author: Alan Woodward +Date: Fri Jun 28 08:19:00 2019 +0100 + + Fix PreConfiguredTokenFilters getSynonymFilter() implementations (#38839) (#43678) + + When we added support for TokenFilterFactories to specialise how they were used when parsing + synonym files, PreConfiguredTokenFilters were set up to either apply themselves, or be ignored. + This behaviour is a leftover from an earlier iteration, and also has an incorrect default. + + This commit makes preconfigured token filters usable in synonym file parsing by default, and brings + those filters that should not be used into line with index-specific filter factories; in indexes created + before version 7 we emit a deprecation warning, and we throw an error in indexes created after. + + Fixes #38793 + +commit 94f18da5df442f3bc4c1497e0bd353c5b75307dd +Author: Przemysław Witek +Date: Fri Jun 28 07:37:21 2019 +0200 + + Add version and create_time to data frame analytics config (#43683) (#43712) + +commit 5b4089e57e5a50871512afb7b42adb6a0354965f +Author: Ryan Ernst +Date: Thu Jun 27 18:45:14 2019 -0700 + + Remove nodeId from BaseNodeRequest (#43658) + + TransportNodesAction provides a mechanism to easily broadcast a request + to many nodes, and collect the respones into a high level response. Each + node has its own request type, with a base class of BaseNodeRequest. + This base request requires passing the nodeId to which the request will + be sent. However, that nodeId is not used anywhere. It is private to the + base class, yet serialized to each node, where the node could just as + easily find the nodeId of the node it is on locally. + + This commit removes passing the nodeId through to the node request + creation, and guards its serialization so that we can remove the base + request class altogether in the future. + +commit 34a86cc321c6841a89fcf74ce26c7cc78b627896 +Author: Benjamin Trent +Date: Thu Jun 27 20:42:43 2019 -0500 + + [ML] Allowing stopped status in HLRC testStartStop (#43710) (#43719) + +commit 3607876a7188eb3a9b972a4c5e5026aee99caed4 +Author: Igor Motov +Date: Thu Jun 27 13:34:33 2019 -0400 + + Geo: Makes coordinate validator in libs/geo plugable (#43657) + + Moves coordinate validation from Geometry constructors into + parser. + + Relates #43644 + +commit ce8771feb75c69601ab5710d6f5c4869ddef140c +Author: Nhat Nguyen +Date: Thu Jun 27 18:27:16 2019 -0400 + + Do not use MockInternalEngine in GatewayIndexStateIT (#43716) + + GatewayIndexStateIT#testRecoverBrokenIndexMetadata replies on the + flushing on shutdown. This behaviour, however, can be randomly disabled + in MockInternalEngine. + + Closes #43034 + +commit 1b7bcdc3a0329e03efb3a8fabdfdb20e717c0fac +Author: Lisa Cawley +Date: Thu Jun 27 15:16:24 2019 -0700 + + [DOCS] Adds data frame API response codes for allow_no_match (#43666) + +commit 42cb59f7b4019d3199a346f36aaf0396ae94ef6c +Author: Lisa Cawley +Date: Thu Jun 27 13:58:42 2019 -0700 + + [DOCS] Updates ML APIs to use new API template (#43711) + +commit 68dbbd87937344e315417c38c060b5c4e6868b58 +Author: Przemysław Witek +Date: Thu Jun 27 18:51:48 2019 +0200 + + Deduplicate two similar TimeUtils classes. (#43697) + + * Deduplicate org.elasticsearch.xpack.core.dataframe.utils.TimeUtils and org.elasticsearch.xpack.core.ml.utils.time.TimeUtils into a common class: org.elasticsearch.xpack.core.common.time.TimeUtils. + * Add unit tests for parseTimeField and parseTimeFieldToInstant methods + +commit d46e2bb26a7a5ee7c2e245a7b142bac9bb8813a8 +Author: lcawl +Date: Thu Jun 27 09:42:47 2019 -0700 + + [DOCS] Adds anchors and attributes to ML APIs + +commit 6744344ef2beaf30d1ca8326463cb898569a21d1 +Author: Yannick Welsch +Date: Thu Jun 27 18:08:48 2019 +0200 + + Handle situation where only voting-only nodes are bootstrapped (#43628) + + Adds support for the situation where only voting-only nodes are bootstrapped. In that case, they will + still try to become elected and bring full master nodes into the cluster. + +commit f39619d18251a5c74c19437b6271e9f4298a06f8 +Author: David Roberts +Date: Thu Jun 27 16:37:00 2019 +0100 + + [ML] Don't write timing stats on no-op (#43680) + + Similar to elastic/ml-cpp#512, if a job opens and closes and + does nothing in between we shouldn't write timing stats to + the results index. + +commit df4b30fd8bfae967c82500f7299f54a63453df11 +Author: Jim Ferenczi +Date: Thu Jun 27 17:00:21 2019 +0200 + + Fix propagation of enablePositionIncrements in QueryStringQueryBuilder (#43578) + + This change fixes the propagation of the enablePositionIncrements option + to the underlying QueryBuilder. + + Closes #43574 + +commit 329d05f61e327160faaf30e9313c25db4ab967fe +Author: Jim Ferenczi +Date: Thu Jun 27 16:56:15 2019 +0200 + + Fix UOE on search requests that match a sparse role query (#43668) + + Search requests executed through the SecurityIndexSearcherWrapper throw + an UnsupportedOperationException if they match a sparse role query. + When low level cancellation is activated (which is the default since #42857), + the context index searcher creates a weight that doesn't handle #scorer. + This change fixes this bug and adds a test to ensure that we check this case. + +commit cd4f81e15e7fcec44a99eeeb025b39fb2f8697ad +Author: Armin Braun +Date: Thu Jun 27 16:51:11 2019 +0200 + + Remove Unused AWS KMS Dependency (#43671) (#43679) + + * We don't make use of KMS at the moment, no need to have this dependency here + +commit c5beb05f77b9951bc81b6ec1aac44c28e54b2a0d +Author: David Roberts +Date: Thu Jun 27 14:32:47 2019 +0100 + + [ML][DataFrame] Consider data frame templates internal in REST tests (#43692) + + The data frame index template pattern was not in the list + considered as internal and therefore not needing cleanup + after every test. + +commit 36360358b2187c9d20a9495ef59ab378b6c1d765 +Author: Christoph Büscher +Date: Thu Jun 27 14:33:28 2019 +0200 + + Move query builder caching check to dedicated tests (#43238) + + Currently `AbstractQueryTestCase#testToQuery` checks the search context cachable + flag. This is a bit fragile due to the high randomization of query builders + performed by this general test. Also we might only rarely check the + "interesting" cases because they rarely get generated when fully randomizing the + query builder. + + This change moved the general checks out ot #testToQuery and instead adds + dedicated cache tests for those query builders that exhibit something other than + the default behaviour. + + Closes #43200 + +commit 4882b932d8acb79713d87e612198cc7067c2cfc4 +Author: Alan Woodward +Date: Thu Jun 27 12:44:29 2019 +0100 + + Issue deprecation warnings when preconfigured delimited_payload_filter is used (#43684) + + #26625 deprecated delimited_payload_filter and added tests to check + that warnings would be emitted when both a normal and pre-configured + filter were used. Unfortunately, due to a bug in the Analyze API, the pre- + configured filter check was never actually triggered, and it turns out that + the deprecation warning was not in fact being emitted in this case. + + #43568 fixed the Analyze API bug, which then surfaced this on backport. + + This commit ensures that the preconfigured filter also emits the warnings + and triggers an error if a new index tries to use a preconfigured + delimited_payload_filter + +commit 8ff5519b11df4768aafa199a9f5ff370584be444 +Author: Alan Woodward +Date: Thu Jun 27 09:01:53 2019 +0100 + + Use preconfigured filters correctly in Analyze API (#43568) + + When a named token filter or char filter is passed as part of an Analyze API + request with no index, we currently try and build the relevant filter using no + index settings. However, this can miss cases where there is a pre-configured + filter defined in the analysis registry. One example here is the elision filter, which + has a pre-configured version built with the french elision set; when used as part + of normal analysis, this preconfigured set is used, but when used as part of the + Analyze API we end up with NPEs because it tries to instantiate the filter with + no index settings. + + This commit changes the Analyze API to check for pre-configured filters in the case + that the request has no index defined, and is using a name rather than a custom + definition for a filter. + + It also changes the pre-configured `word_delimiter_graph` filter and `edge_ngram` + tokenizer to make their settings consistent with the defaults used when creating + them with no settings + + Closes #43002 + Closes #43621 + Closes #43582 + +commit 05a7333eca2f3f7060c68e636f0296acc5be3543 +Author: Alan Woodward +Date: Thu Jun 27 08:56:26 2019 +0100 + + Require [articles] setting in elision filter (#43083) + + We should throw an exception at construction time if a list of + articles is not provided, otherwise we can get random NPEs during + indexing. + + Relates to #43002 + +commit bed7e68014f85507a941d28d1bae64e1190bcfe7 +Author: Julie Tibshirani +Date: Thu Jun 27 08:27:01 2019 +0300 + + Make the ignore_above docs tests more robust. (#43349) + + It is possible for internal ML indices like `.data-frame-notifications-1` to leak, + causing other docs tests to fail when they accidentally search over these + indices. This PR updates the ignore_above tests to only search a specific index. + +commit 683e116601b6369a1a596520ed957d4f322c9c95 +Merge: d6a7fd9f30c ba518722a29 +Author: Martijn van Groningen +Date: Thu Jun 27 08:35:37 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit ba518722a291cabba8fe2aae7aa8fa2a43208edf +Author: Przemysław Witek +Date: Thu Jun 27 08:08:39 2019 +0200 + + [7.x] [ML] Tag destination index with data frame metadata (#43567) (#43660) + +commit ad84059db6b1b3e0d494d38e2ebc69e6fabfd49d +Author: Lisa Cawley +Date: Wed Jun 26 13:46:21 2019 -0700 + + [DOCS] Updates data frame APIs to use API template (#43610) + +commit d05593c3adeacd2dad55ea07521ba981bef93c6f +Author: Benjamin Trent +Date: Wed Jun 26 14:59:19 2019 -0500 + + [ML][Data Frame] adds tests for continuous DF (#43601) (#43654) + +commit 52e26bbc4219598424b3384ba2d2f161d1fe26b3 +Author: Benjamin Trent +Date: Wed Jun 26 13:35:51 2019 -0500 + + [ML][Data Frame] improve pivot nested field validations (#43548) (#43636) + + * [ML][Data Frame] improve pivot nested field validations + + * addressing pr comments + +commit c00e305d791c8dd37ad3c8c9c42aac4e57029e0b +Author: Armin Braun +Date: Wed Jun 26 20:00:42 2019 +0200 + + Optimize Selector Wakeups (#43515) (#43650) + + * Use atomic boolean to guard wakeups + * Don't trigger wakeups from the select loops thread itself for registering and closing channels + * Don't needlessly queue writes + + Co-authored-by: Tim Brooks + +commit 87566c9324642700cc65ec2100238c6b40b4e03a +Author: James Rodewig +Date: Wed Jun 26 13:44:33 2019 -0400 + + [DOCS] Change 'X-Pack APIs' section to 'REST APIs' (#43451) + +commit e1f761dfc7525e4c318308e84164ee39a6e2ffd0 +Author: David Kyle +Date: Wed Jun 26 16:35:14 2019 +0100 + + [Ml Data Frame] Size the GET stats search by number of Ids requested (#43206) + + Set the size of the search request to the number of ids limited by 10,000 + +commit 5035454613809bb9f6052e319c9e29a6f600f2c4 +Author: Yannick Welsch +Date: Wed Jun 26 17:49:15 2019 +0200 + + Remove blank file + + Relates to #43410 + +commit 05b945d0109a4e1d1dfa02bf7cf9e15c4f623070 +Author: Yannick Welsch +Date: Wed Jun 26 17:40:26 2019 +0200 + + Avoid AssertionError when closing engine (#43638) + + Lucene throwing an AlreadyClosedException when closing the engine is fine, and should not trigger + an AssertionError. + + Closes #43626 + +commit 76d0edd1a4e009175fd40155f58d094a57f0b1e1 +Author: Alan Woodward +Date: Wed Jun 26 15:36:47 2019 +0100 + + Add prefix intervals source (#43635) + + This commit adds a prefix intervals source, allowing you to search + for intervals that contain terms starting with a given prefix. The source + can make use of the index_prefixes mapping option. + + Relates to #43198 + +commit c121b00c98b0259a05f5609656469a66a94ee0fa +Author: Benjamin Trent +Date: Wed Jun 26 10:09:56 2019 -0500 + + [7.x] [ML][Data Frame] Add support for allow_no_match for endpoints (#43490) (#43637) + + * [ML][Data Frame] Add support for allow_no_match for endpoints (#43490) + + * [ML][Data Frame] Add support for allow_no_match parameter in endpoints + + Adds support for: + * Get Transforms + * Get Transforms stats + * stop transforms + + * Update DataFrameTransformDocumentationIT.java + +commit 500205e8c57a12483779812655e8422260846560 +Author: Stuart Tettemer +Date: Wed Jun 26 09:06:34 2019 -0600 + + Add painless method getByPath, get value from nested collections with dotted path (#43170) (#43606) + + Given a nested structure composed of Lists and Maps, getByPath will return the value + keyed by path. getByPath is a method on Lists and Maps. + + The path is string Map keys and integer List indices separated by dot. An optional third + argument returns a default value if the path lookup fails due to a missing value. + + Eg. + ['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key1') = ['c', 'd'] + ['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key1.0') = 'c' + ['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key2', 'x') = 'x' + [['key0': 'value0'], ['key1': 'value1']].getByPath('1.key1') = 'value1' + + Throws IllegalArgumentException if an item cannot be found and a default is not given. + Throws NumberFormatException if a path element operating on a List is not an integer. + + Fixes #42769 + +commit 2fa6bc5e12670c68c7c55cb3b80f76ec25a632b9 +Author: Tim Brooks +Date: Wed Jun 26 10:50:14 2019 -0400 + + Properly serialize remote query in ReindexRequest (#43596) + + This commit modifies the RemoteInfo to clarify that a search query + must always be serialized as JSON. Additionally, it adds an assertion + to ensure that this is the case. This fixes #43406. + + Additionally, this PR implements AbstractXContentTestCase for the + reindex request. This is related to #43456. + +commit 51161a4b0e54f9675e4a4d01bcea424f27b21b00 +Author: Jake Landis +Date: Wed Jun 26 08:50:11 2019 -0500 + + add 7.2.0 release notes + +commit 531efb3fe5c5ac06cee10db111ee635d476b29d7 +Author: David Kyle +Date: Wed Jun 26 13:51:41 2019 +0100 + + Remove unreleased 7.1.2 version constant (#43629) + + This was breaking BWC tests as the presence of the constant implied 7.1.2 was released + +commit 83067968caed8fcfa134f878cda8c5c0d063504d +Author: Armin Braun +Date: Wed Jun 26 13:43:32 2019 +0200 + + Add SAS Token Authentication Support to Azure Repo Plugin (#42982) (#43618) + + * Added setting for SAS token + * Added support for the token in tests + * Relates #42117 + +commit 31dc5b7d3a8f217780684c40ec9c421ba020d66b +Author: David Roberts +Date: Wed Jun 26 11:52:06 2019 +0100 + + [TEST] Wait for replicas before stopping nodes in ML distributed test (#43622) + + If we stop a node before replicas exist then + the test can fail because we lose a whole + index if we stop the node with the primary on. + +commit 58d0d5c51bcafaef7b91da32b6207ace6b800295 +Author: David Kyle +Date: Wed Jun 26 10:12:21 2019 +0100 + + Mute DiskDisruptionIT#testGlobalCheckpointIsSafe + + Relates to #43626 + +commit 558e323c8983ac41ae53268831b7cb4d2cb53e68 +Author: David Roberts +Date: Tue Jun 25 16:36:02 2019 +0100 + + [ML] Introduce a setting for the process connect timeout (#43234) + + This change introduces a new setting, + xpack.ml.process_connect_timeout, to enable + the timeout for one of the external ML processes + to connect to the ES JVM to be increased. + + The timeout may need to be increased if many + processes are being started simultaneously on + the same machine. This is unlikely in clusters + with many ML nodes, as we balance the processes + across the ML nodes, but can happen in clusters + with a single ML node and a high value for + xpack.ml.node_concurrent_job_allocations. + +commit 9493c145d7264083f9577e3ad145675b975f6f16 +Author: Alexander Reelsen +Date: Wed Jun 26 10:03:16 2019 +0200 + + Upgrade jcodings dependency to 1.0.44 (#43334) + +commit e6fdbedb031fd8d2641ed34b00e91df447937397 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Jun 26 09:27:58 2019 +0200 + + Reindex remote version lookup test refactor (#43576) + + Refactor test to reuse code and ease maintenance + +commit 2049f715b37accb1a38d27292b2e0a51ca3795be +Author: Yannick Welsch +Date: Tue Jun 25 17:29:30 2019 +0200 + + Add voting-only master node (#43410) + + A voting-only master-eligible node is a node that can participate in master elections but will not act + as a master in the cluster. In particular, a voting-only node can help elect another master-eligible + node as master, and can serve as a tiebreaker in elections. High availability (HA) clusters require at + least three master-eligible nodes, so that if one of the three nodes is down, then the remaining two + can still elect a master amongst them-selves. This only requires one of the two remaining nodes to + have the capability to act as master, but both need to have voting powers. This means that one of + the three master-eligible nodes can be made as voting-only. If this voting-only node is a dedicated + master, a less powerful machine or a smaller heap-size can be chosen for this node. Alternatively, a + voting-only non-dedicated master node can play the role of the third master-eligible node, which + allows running an HA cluster with only two dedicated master nodes. + + Closes #14340 + + Co-authored-by: David Turner + +commit 11f41c4e7d07d5ad1fd850cfd774e4315ef6238c +Author: David Turner +Date: Fri Apr 19 16:02:35 2019 +0100 + + Omit non-masters in ClusterFormationFailureHelper (#41344) + + Today the `ClusterFormationFailureHelper` says `... discovery will continue + using ... from last-known cluster state` and lists all the nodes in the + last-known cluster state. In fact we ignore the master-ineligible nodes in the + last-known cluster state during discovery. This commit fixes this by listing + only the master-eligible nodes from the cluster state in this message. + +commit 480453aa246fdb6676da0bda80562077d453dccc +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Jun 26 14:30:51 2019 +1000 + + Make role descriptors optional when creating API keys (#43481) (#43614) + + This commit changes the `role_descriptors` field from required + to optional when creating API key. The default behavior in .NET ES + client is to omit properties with `null` value requiring additional + workarounds. The behavior for the API does not change. + Field names (`id`, `name`) in the invalidate api keys API documentation have been + corrected where they were wrong. + + Closes #42053 + +commit ca43cdf7559b6ea641b810af7fa2e59c0acda3bf +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Jun 26 13:32:01 2019 +1000 + + Fix for PemTrustConfigTests.testTrustConfigReloadsFileContents failure (#43539) (#43613) + + The test `PemTrustConfigTests.testTrustConfigReloadsFileContents` failed + intermittently with `ArrayIndexOutOfBoundsException` while parsing + the randomly generated bytes array representing DER encoded stream. + This seems to be a bug in JDK (once confirmed we can raise the bug + in JDK bugs system). + + The problem arises when the `X509Factory#parseX509orPKCS7()` tries to + [create `PKCS7` block](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/provider/X509Factory.java#L460) from der encoded stream. While constructing PKCS7 + block it tries to create `ContentInfo` type but fails to do so for the + stream where the length after the DER SEQUENCE is 0. + `DerInputStream#getSequence` [may return empty array of `DerValue`](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/util/DerInputStream.java#L409..L412) but + [the code in `ContentInfo`](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/pkcs/ContentInfo.java#L135) does not check for the empty thereby throwing + `ArrayIndexOutOfBoundsException`. + + Closes #42509 + +commit 05e1f55a8899b284e2cf766ded0e03cdad92e749 +Author: Nhat Nguyen +Date: Tue Jun 25 22:38:55 2019 -0400 + + Ensure relocation target still tracked when start handoff (#42201) + + If the master removes the relocating shard, but recovery isn't aware of + it, then we can enter an invalid state where ReplicationTracker does not + include the local shard. + +commit 58179af5af78e8a9a7e84cab289fd4a089715e36 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Jun 26 12:55:41 2019 +1000 + + Enable Kerberos tests (#43519) (#43612) + + Now that the fix krb5-kdc fixture (entropy problem in docker container) + is in and the converting `kerberos-tests` to testclusters is done, + enabling the kerberos-tests + + Closes #40678 + +commit 9a3c86d4220eb561d95bbed292e8279af5c4b1d2 +Author: Jake Landis +Date: Tue Jun 25 12:53:45 2019 -0500 + + include 7.2.1 as a version (#43584) + +commit 892708198145f7a17483806bcc12cd7aff687250 +Author: Lee Hinman +Date: Tue Jun 25 14:06:23 2019 -0600 + + [7.x] Add TimeValue.toHumanReadableString() to allow specifyin… (#43545) + + * Enhance TimeValue.toString() to allow specifying fractional values. + + This enhances the `TimeValue` class to allow specifying the number of + truncated fractional decimals when calling `toString()`. The default + remains 1, however, more (or less, such as 0) can be specified to change + the output. + + This commit also re-organizes some things in `TimeValue` such as putting + all the class variables near the top of the class, and moving the + constructors to the first methods in the class, in order to follow the + structure of our other code. + + * Rename `toString(...)` to `toHumanReadableString(...)` + +commit 50eac875e45ccdde9901921745c620da1ad00b87 +Author: James Rodewig +Date: Tue Jun 25 15:24:44 2019 -0400 + + [DOCS] Rewrite `range` query (#43282) + +commit e738f0e6d205cf17e331aedf20fabfb4bede9a01 +Author: David Turner +Date: Tue Jun 25 20:04:24 2019 +0100 + + Allow extra time for a warning to be logged (#43597) + + Today we assert that a warning is logged after no more than + `discovery.cluster_formation_warning_timeout`, but the deterministic scheduler + adds a small amount of extra randomness to the timing of future events, causing + the following build to fail: + + ./gradlew :server:test --tests "org.elasticsearch.cluster.coordination.CoordinatorTests.testLogsWarningPeriodicallyIfClusterNotFormed" -Dtests.seed=DF35C28D4FA9EE2D + + This commit adds an allowance for this extra time. + +commit 76a750a0a0eebade443b92903fcdd2c81fac40bf +Author: Przemysław Witek +Date: Tue Jun 25 20:43:38 2019 +0200 + + Remove unused mapStringsOrdered method (#42513) (#43585) + +commit 173338ad37a09e02ead4cffe467440790beebdd2 +Author: Chris Koehnke +Date: Tue Jun 25 14:03:48 2019 -0400 + + Fix dockerfile for non-local builds (#43591) + + Use the `source_elasticsearch` variable to conditionally get the command + needed for release builds for the [dockerfiles repository][0]. + + Fixes https://github.com/elastic/elasticsearch/issues/43590 + + [0]: https://github.com/elastic/dockerfiles + +commit 0dc1c12f134bb82e5d68f2ca57a8fa72ec915b67 +Author: Tanguy Leroux +Date: Tue Jun 25 20:02:34 2019 +0200 + + Fix indices shown in _cat/indices (#43286) + + After two recent changes (#38824 and #33888), the _cat/indices API + no longer report information for active recovering indices and + non-replicated closed indices. It also misreport replicated closed + indices that are potentially not authorized for the user. + + This commit changes how the cat action works by first using the + Get Settings API in order to resolve authorized indices. It then uses + the Cluster State, Cluster Health and Indices Stats APIs to retrieve + information about the indices. + + Closes #39933 + +commit d6a7fd9f30c1fb27f4e20e24a955b8443c1bd279 +Author: Martijn van Groningen +Date: Tue Jun 25 19:53:35 2019 +0200 + + unmuted test + +commit 126c2fd2d59d34ab02bbba111db781492a5e5ae6 +Author: Dimitris Athanasiou +Date: Tue Jun 25 20:29:11 2019 +0300 + + [7.x][ML] Machine learning data frame analytics (#43544) (#43592) + + This merges the initial work that adds a framework for performing + machine learning analytics on data frames. The feature is currently experimental + and requires a platinum license. Note that the original commits can be + found in the `feature-ml-data-frame-analytics` branch. + + A new set of APIs is added which allows the creation of data frame analytics + jobs. Configuration allows specifying different types of analysis to be performed + on a data frame. At first there is support for outlier detection. + + The APIs are: + + - PUT _ml/data_frame/analysis/{id} + - GET _ml/data_frame/analysis/{id} + - GET _ml/data_frame/analysis/{id}/_stats + - POST _ml/data_frame/analysis/{id}/_start + - POST _ml/data_frame/analysis/{id}/_stop + - DELETE _ml/data_frame/analysis/{id} + + When a data frame analytics job is started a persistent task is created and started. + The main steps of the task are: + + 1. reindex the source index into the dest index + 2. analyze the data through the data_frame_analyzer c++ process + 3. merge the results of the process back into the destination index + + In addition, an evaluation API is added which packages commonly used metrics + that provide evaluation of various analysis: + + - POST _ml/data_frame/_evaluate + +commit 1b902aa746d691ea1d5a461a4f95fed5c2a6fa2b +Author: James Baiera +Date: Tue Jun 25 11:16:30 2019 -0400 + + Make enrich processor use search action through a client (#43311) + + Add client to processor parameters in the ingest service. + Remove the search provider function from the processor parameters. + ExactMatchProcessor and Factory converted to use client. + Remove test cases that are no longer applicable from processor. + +commit b5987011984e3fd2c2f53ec0a4654dca3e1e0ef6 +Author: James Rodewig +Date: Tue Jun 25 12:34:18 2019 -0400 + + [DOCS] Add redirect for painless examples anchor + +commit a38f509284575e8da07854f1efc946a7f215017e +Author: Alpar Torok +Date: Tue Jun 25 18:46:52 2019 +0300 + + Testclusters: convert left-overs from checkPart1 (#43370) + + * Testclusters: convert left-overs from checkPart1 + +commit 63fef5a31e4aae8ea11edda892d8873fbb7e543c +Author: Zachary Tong +Date: Tue Jun 25 11:50:19 2019 -0400 + + Add scripting support to AggregatorTestCase (#43494) + + This refactors AggregatorTestCase to allow testing mock scripts. + The main change is to QueryShardContext. This was previously mocked, + but to get the ScriptService you have to invoke a final method + which can't be mocked. + + Instead, we just create a mostly-empty QueryShardContext and populate + the fields that are needed for testing. It also introduces a few + new helper methods that can be overridden to change the default + behavior a bit. + + Most tests should be able to override getMockScriptService() to supply + a ScriptService to the context, which is later used by the aggs. + More complicated tests can override queryShardContextMock() as before. + + Adds a test to MaxAggregatorTests to test out the new functionality. + +commit 2beb1933110a747209eeac66c7473ee2e324f209 +Author: Alpar Torok +Date: Tue Jun 25 18:45:07 2019 +0300 + + Disable testing conventions on Windows (#43532) + + Tests are disabled on Windows. Conventions also need to be disabled. + +commit 09695decb341dc7d19ea8e86eff4d22322a40448 +Author: Alpar Torok +Date: Tue Jun 25 18:33:42 2019 +0300 + + Fix failing LicensingDocumentationIT test (#43533) + + This PR brings corrections for cluster name after migrating to testclusters. + Not sure how this slipped trough the cracks when converting. + + Closes #43504 + +commit 970e157eac73e6da41e1a3d11dbee0a39c067318 +Author: Benjamin Trent +Date: Tue Jun 25 10:09:39 2019 -0500 + + [ML][Data Frame] Adjusting error message (#43455) (#43580) + + * Adjusting error message + + * Update TransportPutDataFrameTransformAction.java + + * Update TransportPutDataFrameTransformAction.java + +commit 66693c27069f3e2e38b98e2b70a6b1de8781db07 +Author: rbayet <2353374+rbayet@users.noreply.github.com> +Date: Tue Jun 25 16:32:04 2019 +0200 + + Fixing backquote in fail_on_unsupported_field (#43572) + +commit c702cd741521bf8573d8b81836fc6a3919a90e4f +Author: Przemysław Witek +Date: Tue Jun 25 16:04:54 2019 +0200 + + [7.x] Implement XContentParser.genericMap and XContentParser.genericMapOrdered methods (#42059) (#43575) + +commit b15e40ffada6c56ee15d0e8b7d76b6830918aed1 +Author: Przemysław Witek +Date: Tue Jun 25 15:48:39 2019 +0200 + + Extract TimingStats-related functionality into TimingStatsReporter (#43371) (#43557) + +commit c594a956e29cb0d5f095f2f3e9d5995ed22dc63f +Author: Ernesto Reig +Date: Tue Jun 25 14:48:14 2019 +0200 + + Default number of shards is now 1 instead of 5 (#43573) + + As specified in the [Breaking changes for 7.X](https://www.elastic.co/guide/en/elasticsearch/reference/7.1/breaking-changes-7.0.html#breaking_70_indices_changes), the default number of shards for an index is now `1` instead of `5`. + +commit 67a67ac3ce68a03649db1ce8a485f5b5cd950635 +Author: Przemyslaw Gomulka +Date: Tue Jun 25 14:35:24 2019 +0200 + + Fix DieWithDignity logs assertion backport(#43543) #43562 + + This test is likely to kill the server in the middle of writing logs. This means that we can end up with logs with partially written json log lines and standard json parsers would fail on this. + This fix is to use regular expressions on json logs.(just like the previous approach on plain text logs) + closes #43413 + +commit bfd82012e8be83a506bdfc3ff72a5019a3fc8d7c +Author: Benjamin Trent +Date: Tue Jun 25 07:29:44 2019 -0500 + + [ML][Data Frame] fixing some data frame hlrc tests (#43446) (#43491) + + * [ML][Data Frame] fixing some data frame hlrc tests + + * adding task|indexer state checks back + +commit 36f0e8a8bb13e2564149d5cfcfe58eef47a55472 +Author: Martijn van Groningen +Date: Tue Jun 25 13:21:47 2019 +0200 + + Added multi node enrich tests and fixed serialization issues. (#43386) + + The test for now tests the enrich APIs in a multi node environment. + Picked EsIntegTestCase test over a real qa module in order to avoid + adding another module that starts a test cluster. + +commit 9c285ddbaba321983d5dfb6b0050abc9fbd987c9 +Author: David Roberts +Date: Tue Jun 25 11:56:46 2019 +0100 + + [ML] Improve message when native controller cannot connect (#43565) + + The error message if the native controller failed to run + (for example due to running Elasticsearch on an unsupported + platform) was not easy to understand. This change removes + pointless detail from the message and adds some hints about + likely causes. + + Fixes #42341 + +commit b4ed7f463a3fe31eeb5cee7b9326fa529891c699 +Author: Armin Braun +Date: Tue Jun 25 13:04:43 2019 +0200 + + Fix CreateRepository Requeset in HLRC (#43522) (#43566) + + * verify = false is the non-default case for this request -> adjusted the code accordingly and expanded the test to cover this case + * Closes #43521 + +commit 62a28921e84de79d25828911a0214ac7e0ccdc3b +Author: Armin Braun +Date: Tue Jun 25 13:04:04 2019 +0200 + + Cleanup IndicesService#CacheCleaner Scheduling (#42060) (#43528) + + * Follow up to #42016 + +commit 3d5e4577aa47713a3a4375279c87832d5e7cb770 +Author: Yannick Welsch +Date: Tue Jun 25 12:48:20 2019 +0200 + + Fix testPostOperationGlobalCheckpointSync + + The conditions in this test do not hold true anymore after #43205. + + Relates to #43205 + +commit e4738587c0d7d418b1c880d99dec1d1c2cc8f9da +Author: Przemysław Witek +Date: Fri May 10 07:02:34 2019 +0200 + + Implement factory methods for ValidationException (#41993) + + Implement factory methods for ValidationException to make the client code more concise (1 LOC vs 3 LOC for a single error scenario) + +commit d0634e444d71ef351b2d045490eae60e50e058aa +Author: Martijn van Groningen +Date: Tue Jun 25 10:09:29 2019 +0200 + + Fixed compile errors after merge. + +commit f587519f17e2bf1511b4c40b3f45267c2c4b030f +Merge: c0d5ec87e13 8d9a971259b +Author: Martijn van Groningen +Date: Tue Jun 25 10:09:51 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 8d9a971259bdbcec0a68894c64bea61675f05ada +Author: Lee Hinman +Date: Mon Jun 24 17:16:29 2019 -0600 + + Revert "Test clusters: convert x-pack qa tests (#43283)" (#43549) + + This reverts commit ccaa8c33ba7383b2abf49af685135dd0eb0d0e54. + +commit df42fac9ace45f4df2d5a2d09ae9417cfbc46831 +Author: debadair +Date: Mon Jun 24 15:29:47 2019 -0700 + + [DOCS] Edited title/subtitle. (#43552) + +commit 01205432fe38189c96eb564dc77b086e13015d7e +Author: Nhat Nguyen +Date: Mon Jun 24 17:12:57 2019 -0400 + + Unmute testOpenCloseApiWildcards + + Relates #39578 + +commit ae31ca5f7e475ba70e7f5b867bb83b5b8c722481 +Author: Jim Ferenczi +Date: Mon Jun 24 21:31:32 2019 +0200 + + Fix score mode of the MinimumScoreCollector (#43527) + + This change fixes the score mode of the minimum score collector to + be set based on the score mode of the child collector (top docs). + + Closes #43497 + +commit c0d5ec87e13fce9e85e7cb20ff3e1e99ded61b0a +Author: James Baiera +Date: Mon Jun 24 13:54:43 2019 -0400 + + Set enrich indices to be read only before swapping their aliases (#42874) + +commit 8ffd9c6981edcc9f24b672569dc223c0c747154c +Author: Lisa Cawley +Date: Mon Jun 24 10:14:12 2019 -0700 + + [DOCS] Adds administering section (#43493) + +commit d45f12799ccaf55ae65c28524717f6b348c2423d +Author: Yannick Welsch +Date: Mon Jun 24 18:35:20 2019 +0200 + + Sync global checkpoint on pending in-sync shards (#43526) + + At the end of a peer recovery the primary wants to mark the replica as in-sync. For that the + persisted local checkpoint of the replica needs to have caught up with the global checkpoint on the + primary. If translog durability is set to ASYNC, this means that information about the persisted local + checkpoint can lag on the primary and might need to be explicitly fetched through a global + checkpoint sync action. Unfortunately, that action will only be triggered after 30 seconds, and, even + worse, will only run based on what the in-sync shard copies say (see + IndexShard.maybeSyncGlobalCheckpoint). As the replica has not been marked as in-sync yet, it is + not taken into consideration, and the primary might have its global checkpoint equal to the max seq + no, so it thinks nothing needs to be done. + + Closes #43486 + +commit 97cd417829cdf43ed83583e2a8dce68285a7043c +Author: Ryan Ernst +Date: Mon Jun 24 09:34:32 2019 -0700 + + Distribution download plugin cleanup (#43461) + + This commit addresses some post-review comments. + + relates #43247 + +commit 5eb044e6353a7b37819e872901895be316e53d23 +Author: Jack Conradson +Date: Mon Jun 24 09:00:44 2019 -0700 + + Add annotations to Painless whitelist (#43239) + + This change adds the ability to attach annotative information for + classes, methods, fields, static methods, class bindings, and + instance bindings during Painless whitelisting. + + Annotations are specified as @annotation or optionally as + @annotation[parameter="argument",...]. + + Annotations open up the ability to specify whitelist objects as + having a short name (no_import -> @no_import) or deprecated. + +commit eaa9ee1f166d7b13a2a6b814aefc7319376e60d6 +Author: Zachary Tong +Date: Mon Jun 24 12:19:46 2019 -0400 + + Set document on script when using Bytes.WithScript (#43390) + + Long and Double ValuesSource set the current document on the script + before executing, but Bytes was missing this method call. That meant + it was possible to generate an OutOfBoundsException when using + a "value" script (field + script) on keyword or other bytes + fields. + + This adds in the method call, and a few yaml tests to verify correct + behavior. + +commit 6728e63619f3f30a1af42be5e8a73c1cf2a6207b +Author: David Roberts +Date: Mon Jun 24 16:56:19 2019 +0100 + + [DOCS] Rename "job" to "transform" in data frame transform docs (#43534) + +commit 38516a4dd5976e14ce6989458c4a60c25f7632c1 +Author: Tim Brooks +Date: Mon Jun 24 11:35:46 2019 -0400 + + Move nio ip filter rule to be a channel handler (#43507) + + Currently nio implements ip filtering at the channel context level. This + is kind of a hack as the application logic should be implemented at the + handler level. This commit moves the ip filtering into a channel + handler. This requires adding an indicator to the channel handler to + show when a channel should be closed. + +commit fac7efba9ab8639f7815cdf63c23b6198e4aef7b +Author: Gordon Brown +Date: Mon Jun 24 10:02:49 2019 -0600 + + [7.x] Account for node versions during allocation in ILM Shrink (#43300) + + This commit ensures that ILM's Shrink action will take node versions into + account when choosing which node to allocate to when shrinking an + index. Prior to this change, ILM could pick a node with a lower version + than some shards are already allocated to, which causes the new + allocation to fail as shards can't be relocated onto a node with a lower + version than they are already on. + + As part of this, when making the decision about which node to allocate + to prior to Shrink, all shards in the index are considered, rather than + choosing a random shard to consider. + + Further, the unit tests for the logic that chooses a node to allocate + shards to pre-shrink has been improved to validate the behavior in more + realistic and varied initial conditions. + +commit 98d7d231bbcddabc45a9c95348d85a8e56deb927 +Author: Andrey Ershov +Date: Mon Jun 24 17:52:49 2019 +0300 + + Fix testNoMasterActions (#43471) + + This commit performs the proper restore of network disruption. + Previously disruptionScheme.stopDisrupting() was called that does not + ensure that connectivity between cluster nodes is restored. The test + was checking that the cluster has green status, but it was not checking + that connectivity between nodes is restored. + Here we switch to internalCluster().clearDisruptionScheme(true) which + performs both checks before returning. + Similar to #42798 + Closes #42051 + + (cherry picked from commit cd1ed662f847a0055ede7dfbd325e214ec4d1490) + +commit 6945e5d5e6f64d41e886b5424f27642bc3c1df24 +Author: Michael Basnight +Date: Mon Jun 24 10:39:59 2019 -0500 + + Add role for enrich processor (#42677) + + This commit adds the manage_enrich privilege, which grants access to all + of the enrich processor lifecycle actions. In addition this commit also + creates a role which grants access to the generated indices. + + Relates #41939 + + Co-authored-by: Martijn van Groningen + +commit 9794409ca0780c67d8d22fad7232e96f8dc3558f +Author: Tanguy Leroux +Date: Mon Jun 24 16:19:15 2019 +0200 + + Fix broken link + +commit a4dfa7c29ba29334c3215ea34d1f4d08d31586e2 +Author: Tanguy Leroux +Date: Mon Jun 24 15:52:41 2019 +0200 + + Add release highlight for replicated closed indices on 7.2.0 (#43530) + +commit 813551e070fc1455d8db555f33cb128e3e70d65a +Author: Mayya Sharipova +Date: Mon Jun 24 09:22:48 2019 -0400 + + Fix eclipse build gradle for vectors project + + Closes #43496 + +commit 101cf384ba87e993f18f65d2eb7b46e632bfc8a6 +Author: Martijn van Groningen +Date: Mon Jun 24 13:47:37 2019 +0200 + + Replace Streamable w/ Writable in AcknowledgedResponse and subclasses (backport 7.x) (#43525) + + This commit replaces usages of Streamable with Writeable for the + AcknowledgedResponse and its subclasses, plus associated actions. + + Note that where possible response fields were made final and default + constructors were removed. + + This is a large PR, but the change is mostly mechanical. + + Relates to #34389 + Backport of #43414 + +commit 41ebaf57b53139dde4ac9471b9292581416f9281 +Author: Tanguy Leroux +Date: Mon Jun 24 12:59:41 2019 +0200 + + Do not hang on unsupported HTTP methods (#43362) + + Unsupported HTTP methods are detected during requests dispatching + which generates an appropriate error response. Sadly, this error is + never sent back to the client because the method of the original + request is checked again in DefaultRestChannel which throws again + an IllegalArgumentException that is never handled. + + This pull request changes the DefaultRestChannel so that the latest + exception is swallowed, allowing the error message to be sent back + to the client. It also eagerly adds the objects to close to the toClose + list so that resources are more likely to be released if something + goes wrong during the response creation and sending. + +commit 19520d4640ece2178bcc66756efce3c728bcb186 +Author: Yannick Welsch +Date: Mon Jun 24 12:28:59 2019 +0200 + + Add additional logging for #43034 + + It's unclear why sometimes the shard is not flushed on closing + +commit 127a608147198075d92011e2c8c1dbd2e8f8661b +Author: Yannick Welsch +Date: Mon Jun 24 11:38:13 2019 +0200 + + Assert that NOOPs must succeed (#43483) + + We currently assert that adding deletion tombstones to Lucene must always succeed if it's not a + tragic exception, and the same should also hold true for NOOP tombstones. We rely on this + assumption, as without this, we have the risk of creating gaps in the history, which will break + operation-based recoveries and CCR. + +commit ccaa8c33ba7383b2abf49af685135dd0eb0d0e54 +Author: Alpar Torok +Date: Mon Jun 24 12:13:38 2019 +0300 + + Test clusters: convert x-pack qa tests (#43283) + +commit b4f5882fbe1a12086366f232edf3a498983f7754 +Author: Alpar Torok +Date: Mon Jun 24 12:03:19 2019 +0300 + + Improove error on waiting for cluster to come up (#43416) + + Always include all messages from exceptions to make the reason for + failure more visible. + +commit ea44da6069d0ebd321901c57350f5bb60b04a01a +Author: Alpar Torok +Date: Thu Jun 20 09:21:29 2019 +0300 + + Testclusters: conver remaining x-pack (#43335) + + Convert x-pack tests + +commit 0bcadbf846002f21e21ef2d1c2b5547a2f08dd6c +Author: Matthew Adams +Date: Mon Jun 24 09:12:40 2019 +0100 + + Clarify storage location of ML Snapshots (#43437) + + The existing language was misleading about the model snapshots and where they are located. Saying "to disk" sounds like files external to Elasticsearch IMO. It raises the obvious question, where on disk? which node? Is it in the Elasticsearch snapshot repo? The model snapshots are held in an internal index. + +commit eb01208672348cd7829ae40e82f1a97b49587ea0 +Author: Ryan Ernst +Date: Sun Jun 23 23:25:39 2019 -0700 + + Fix the bundled jdk flag to be passed through windows startup (#43502) + + This commit fixes a typo in elasticsearch.bat that prevented the windows + distribution from knowing whether it is using the bundled jdk. + +commit 04bc754d8d1eaeb920c00e220de5ed508422ef7a +Author: Nhat Nguyen +Date: Sun Jun 23 11:29:08 2019 -0400 + + Cleanup legacy logic in CombinedDeletionPolicy (#43484) + + This change removes the support for pre-v6 index commits which + do not have sequence numbers. + +commit 5f87caa54cb90b7f478a6181aa5c4519bbe03283 +Author: Armin Braun +Date: Fri Jun 21 21:28:58 2019 +0200 + + Assert ServerSocketChannel is not Blocking (#43479) (#43488) + + * Assert ServerSocketChannel is not Blocking + + * Relates #43387 which appears to run into blocking accept calls + +commit df9f06213dcffb4a51767ae72b256f45d729d6d1 +Merge: 9de4e878f74 186c3122be7 +Author: Martijn van Groningen +Date: Fri Jun 21 19:58:04 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 186c3122be7cdacc1942905f6c60ae80a7f0f278 +Author: Luca Cavanna +Date: Fri Jun 21 19:05:53 2019 +0200 + + [TEST] Embed msearch samples in MultiSearchRequestTests (#43482) + + Depending on git configuration, line feed on checked out files may be + platform dependent, which causes problems to some msearch tests as the + line separator must always be `/n`. With this change we move two files + to the test code so that we control exactly what line separator is used, + given that the corresponding tests fail on windows. + + Closes #43464 + +commit 6162471d2e45b2a76c3c2e3c8322a3eaf775cfb9 +Author: Igor Motov +Date: Thu Jun 20 05:15:59 2019 -0700 + + Docs: Add description of the coerce parameter in geo_shape mapper (#43340) + + Explains the effect of the coerce parameter on the geo_shape field. + + Relates #35059 + +commit 014fd19abdb54b217deff8185c83540a47ae7e29 +Author: James Rodewig +Date: Fri Jun 21 12:03:19 2019 -0400 + + [DOCS] Rewrite `constant_score` query (#43374) + +commit 359b103f87ca6636967fec1951260dd545c8b7aa +Author: James Rodewig +Date: Fri Jun 21 11:53:01 2019 -0400 + + [DOCS] Rewrite term-level queries overview (#43337) + +commit f4b75d6d14e1943559ddb38a7683add319343dfd +Author: Benjamin Trent +Date: Fri Jun 21 09:11:44 2019 -0500 + + [7.x] [ML][Data Frame] Add version and create_time to transform config (#43384) (#43480) + + * [ML][Data Frame] Add version and create_time to transform config (#43384) + + * [ML][Data Frame] Add version and create_time to transform config + + * s/transform_version/version s/Date/Instant + + * fixing getter/setter for version + + * adjusting for backport + +commit e4fd0ce7301ac7fafab554ba209b435a60b8c35e +Author: David Turner +Date: Fri Jun 21 06:59:14 2019 +0100 + + Reduce TestLogging usage in DisruptionIT tests (#43411) + + Removes `@TestLogging` annotations in `*DisruptionIT` tests, so that the only + tests with annotations are those with open issues. Also adds links to the open + issues in the remaining cases. + + Relates #43403 + +commit eeb1812510cba02e02df289e36276fa92ce408ea +Author: Luiz Guilherme Pais dos Santos +Date: Fri Jun 21 10:24:33 2019 -0300 + + Example of how to set slow logs dynamically per-index (#42384) + + * Example of how to set slow logs dynamically per-index + + * Make _settings API example more explicit + + Co-Authored-By: James Rodewig + + * Add TEST directive to fix CI + + Co-Authored-By: James Rodewig + +commit 4fe650c9e56d5f68ae9d0b0ca802d0827346c5c2 +Author: Christoph Büscher +Date: Fri Jun 21 14:29:46 2019 +0200 + + Fix DefaultShardOperationFailedException subclass xcontent serialization (#43435) + + The current toXContent implementation can fail when the superclasses toXContent + is called (see #43423). This change makes sure that + DefaultShardOperationFailedException#toXContent is final and implementations + need to add special fields in #innerToXContent. All implementations should write + to self-contained xContent objects. Also adding a test for xContent deserialization + to CloseIndexResponseTests. + + Closes #43423 + +commit 73221d2265eb333405970e9e7f56f258e86170cc +Author: David Kyle +Date: Fri Jun 21 10:14:58 2019 +0100 + + [ML] Resolve NetworkDisruptionIT (#43441) + + After the network disruption a partition is created, + one side of which can form a cluster the other can't. + Ensure requests are sent to a node on the correct side + of the cluster + +commit d1280339a828d385f32256afd3b7296b142f2e15 +Author: David Kyle +Date: Fri Jun 21 10:15:51 2019 +0100 + + specifies which index to search in docs for various queries (#43307) (#43428) + + the geo-bounding-box and phrase-suggest docs were susceptible to + failing due to other indices in the cluster. This change restricts + the queries to the index that is set up for the test. + + relates to #43271. + +commit 424ef4f158e1c4a57b2131b83657108a1d0d4ce3 +Author: Simon Willnauer +Date: Fri Jun 21 09:20:39 2019 +0200 + + SecurityIndexSearcherWrapper doesn't always carry over caches and similarity (#43436) + + If DocumentLevelSecurity is enabled SecurityIndexSearcherWrapper doesn't carry + over the cache, cache policy and similarity from the incoming searcher. + +commit 059eb5510867eeee97c67ad42b6217687ff69c7c +Author: Tim Vernum +Date: Fri Jun 21 17:11:07 2019 +1000 + + Use SecureString for password length validation (#43465) + + This replaces the use of char[] in the password length validation + code, with the use of SecureString + + Although the use of char[] is not in itself problematic, using a + SecureString encourages callers to think about the lifetime of the + password object and to clear it after use. + + Backport of: #42884 + +commit c88f2f23a543ae8749c560f6cea2d970d8ff0d6a +Author: Yu +Date: Fri Jun 21 15:02:23 2019 +0800 + + Make Recovery API support `detailed` params (#29076) + + Properly forwards the `detailed` parameter to show the recovery stats details. + + Closes #28910 + +commit 90e151edeb3eb47ab9f1992ba1408e6b2232d38d +Author: Andrei Stefan +Date: Fri Jun 21 08:38:21 2019 +0300 + + Mute MultiSearchRequestTests.java tests (#43467) + +commit cc6c114cb806e98b39196aad29cd5e559cd59531 +Author: Jim Ferenczi +Date: Thu Jun 20 22:01:18 2019 +0200 + + Fix round up of date range without rounding (#43303) + + Today when searching for an exclusive range the java date math parser rounds up the value + with the granularity of the operation. So when searching for values that are greater than + "now-2M" the parser rounds up the operation to "now-1M". This behavior was introduced when + we migrated to java date but it looks like a bug since the joda math parser rounds up values + but only when a rounding is used. So "now/M" is rounded to "now-1ms" (minus 1ms to get the largest inclusive value) + in the joda parser if the result should be exclusive but no rounding is applied if the input + is a simple operation like "now-1M". This change restores the joda behavior in order to have + a consistent parsing in all versions. + + Closes #43277 + +commit 827f8fcbd5addb0030e76f2f3bad8ce542f2451f +Author: Tim Brooks +Date: Thu Jun 20 17:49:11 2019 -0400 + + Move reindex request parsing into request (#43450) + + Currently the fromXContent logic for reindex requests is implemented in + the rest action. This is inconsistent with other requests where the + logic is implemented in the request. Additionally, it requires access to + the rest action in order to parse the request. This commit moves the + logic and tests into the ReindexRequest. + +commit 7b0a259b2cd9c8da3da9ed44605b4c24bbbda93c +Author: Ryan Ernst +Date: Thu Jun 20 14:27:07 2019 -0700 + + Clarify unsupported secure settings behavior (#43454) + + This commit tweaks the docs for secure settings to ensure the user is + aware adding non secure settings to the keystore will result in + elasticsearch not starting. + + fixes #43328 + + Co-Authored-By: James Rodewig + +commit 21515b9ff189e92b952d0b9bac15af5be084008e +Author: Armin Braun +Date: Thu Jun 20 22:31:59 2019 +0200 + + Fix IpFilteringIntegrationTests (#43019) (#43434) + + * Increase timeout to 5s since we saw 500ms+ GC pauses on CI + * closes #40689 + +commit 1053a89b7979081a232287bfe9701f3b56c069f9 +Author: Armin Braun +Date: Thu Jun 20 22:31:36 2019 +0200 + + Log Blocked IO Thread State (#43424) (#43447) + + * Let's log the state of the thread to find out if it's dead-locked or just stuck after being suspended + * Relates #43392 + +commit 6b1e45b5b35f18ec24b3d43a691bbe286563e591 +Author: Deb Adair +Date: Thu Jun 20 12:58:11 2019 -0700 + + [DOCS] Updated the URL for starting in the cloud. + +commit cf610b5e8141a5ea32f789e694d977846c9aba2f +Author: sandmannn +Date: Thu Jun 20 19:19:19 2019 +0200 + + Added parsing of erroneous field value (#42321) + +commit 6e7a0e1b2ada9951539acecd3e924ba76836fd50 +Author: Armin Braun +Date: Thu Jun 20 20:23:51 2019 +0200 + + Remove Azure Container Exists Check (#43288) (#43433) + + * This check is redundant, if the container doesn't exist subsequent operations will fail anyway. Since we are not running this exists check during verification I don't think there's much point to having it in snapshot initialization. + * This PR is mainly motivated by the fact that this forces more permissions to be available in shared environments + +commit 2319fe74c3707d027eb0aed11615f2d85324c84c +Author: debadair +Date: Thu Jun 20 10:33:54 2019 -0700 + + [DOCS] Fixed path to install directory. (#43443) + +commit 2f2d0a198fe7c2b75b16fff4436781ae967490f8 +Author: Jake Landis +Date: Thu Jun 20 12:07:18 2019 -0500 + + add version 6.8.2 + +commit 29d76baf7deaa808467144905b1dc59dbdaf9035 +Author: Yannick Welsch +Date: Thu Jun 20 19:05:45 2019 +0200 + + Increase timeout for assertSeqNos + + Helps with tests that do async translog syncing + +commit 7ef62a6cbc261dc5e95c8d00cd4b23e4dbd05854 +Author: Jack Conradson +Date: Thu Jun 20 08:48:01 2019 -0700 + + Remove old Painless doc generator (#43404) + + This removes the previous Painless API Doc Generator prior to contexts + existing. It has been replaced with the new doc generator that uses the + documentation rest API. + +commit a8a81200d052e038fffa0edb6d611261f711648a +Author: Zachary Tong +Date: Thu Jun 20 11:06:13 2019 -0400 + + Better support for unmapped fields in AggregatorTestCase (#43405) + + AggregatorTestCase will NPE if only a single, null MappedFieldType + is provided (which is required to simulate an unmapped field). While + it's possible to test unmapped fields by supplying other, non-related + field types... that's clunky and unnecessary. AggregatorTestCase + just needs to filter out null field types when setting up. + +commit 8c856d6d913e37416ca242d42852bc03947075a4 +Author: Yannick Welsch +Date: Thu Jun 20 17:29:01 2019 +0200 + + Adapt local checkpoint assertion + + With async durability, it does not hold true anymore after #43205. This is fine. + +commit 5f8db95d60794e908606cd40769e473764b78fa5 +Author: Lisa Cawley +Date: Thu Jun 20 07:22:18 2019 -0700 + + [DOCS] Describe setup for monitoring logs (#42655) + +commit 7b740b4ea3dae634e0919d5420baf354b5d2c4de +Author: debadair +Date: Thu Jun 20 07:49:50 2019 -0700 + + [DOCS] Add brew install instructions. Closes #42914 (#42915) + +commit 12bc38d9e6a225a241e920240f4ff0b92fd52177 +Author: David Kyle +Date: Thu Jun 20 15:35:47 2019 +0100 + + Mute put-transform docs test + + Relates to #43271 + +commit b7322812e00652822bea2d36bc4c2c5b0db040eb +Author: Armin Braun +Date: Thu Jun 20 16:43:42 2019 +0200 + + Upgrade AWS SDK to Latest Version (#42708) (#43422) + + * Just staying up to data on the SDK version + * Use `AbstractAmazonEC2` to shorten code + +commit 99a44a04f7a7bca7f1d487ac38b4897a6b15d72e +Author: Armin Braun +Date: Thu Jun 20 16:38:28 2019 +0200 + + Fix Infinite Loops in ExceptionsHelper#unwrap (#42716) (#43421) + + * Fix Infinite Loops in ExceptionsHelper#unwrap + + * Keep track of all seen exceptions and break out on loops + * Closes #42340 + +commit 39fef8379bf2499e695090683b99c478e542c486 +Author: Armin Braun +Date: Thu Jun 20 16:36:40 2019 +0200 + + Fix FsRepositoryTests.testSnapshotAndRestore (#42925) (#43420) + + * The commit generation can be 3 or 2 here -> fixed by checking the actual generation on the second commit instead of hard coding 2 + * Closes #42905 + +commit 7d1983a7e3ec01c2553eecf301a19d85bafd8a1d +Author: Armin Braun +Date: Thu Jun 20 16:36:17 2019 +0200 + + Fix Operation Timestamps in Tests (#43155) (#43419) + + * For the issue in #43086 we were running into inactive shards because the random timestamps previously used would randomly make `org.elasticsearch.index.shard.IndexShard#checkIdle` see an incorrect+huge inactive time + * Also fixed one other spot in tests that passed `ms` instead of `ns` for the same timestamp on an index op to correctly use relative `ns` + * Closes #43086 + +commit 21e74dd7d266f6ff6619669eb7b0aa8d680dab55 +Author: Armin Braun +Date: Thu Jun 20 16:35:54 2019 +0200 + + Upgrade GCS Repository Dependencies (#43142) (#43418) + + * Upgrade to latest GCS SDK and transitive dependencies (I chose the later version here on conflict) + * Remove now unnecessary hack for custom endpoints (the linked bugs were both resolved in the SDK) + +commit b4c4018d00a21dc9564c80b9851e5aa761fd58bd +Author: synical +Date: Thu Jun 20 15:01:46 2019 +0100 + + Remove Confusing Comment (#43400) + +commit c8eb09f15850f96a58ca265b107d924419a45c7f +Author: David Turner +Date: Thu Jun 20 14:44:13 2019 +0100 + + Fail connection attempts earlier in tests (#43320) + + Today the `DisruptibleMockTransport` always allows a connection to a node to be + established, and then fails requests sent to that node such as the subsequent + handshake. Since #42342, we log handshake failures on an open connection as a + warning, and this makes the test logs rather noisy. This change fails the + connection attempt first, avoiding these unrealistic warnings. + +commit 5af9387fad3b4867015015ef4d3a27b9bc8aad9b +Author: Armin Braun +Date: Wed Jun 5 11:06:05 2019 +0200 + + Fix Stuck IO Thread Logging Time Precision (#42882) + + * The precision of the timestamps we get from the cached time thread is only 200ms by default resulting in a number of needless ~200ms slow network thread execution logs + * Fixed by making the warn threshold a function of the precision of the cached time thread found in the settings + +commit 84d3de1feea96804f180a62c71fe3ec616bbe170 +Author: Andrei Stefan +Date: Thu Jun 20 15:10:21 2019 +0300 + + Mute DieWithDignityIT test. (#43415) + +commit e04a2258fcfa525443b23940152ab341b56526bb +Author: Yannick Welsch +Date: Thu Jun 20 13:58:05 2019 +0200 + + Fix testGlobalCheckpointSync + + The test needed adaption after #43205, as the ReplicationTracker now distinguishes between the + knowledge of the persisted global checkpoint and the computed global checkpoint on the primary + + Follow-up to #43205 + +commit a76c034866f80f430f8370371fd24b2f4b27d446 +Author: Yannick Welsch +Date: Thu Jun 20 13:22:42 2019 +0200 + + Reduce shard started failure logging (#43330) + + If the master is stepping or shutting down, the error-level logging can cause quite a bit of noise. + +commit 7f8e1454ab3283ef361966faa2a02c7bba476da0 +Author: Yannick Welsch +Date: Thu Jun 20 08:46:30 2019 +0200 + + Advance checkpoints only after persisting ops (#43205) + + Local and global checkpoints currently do not correctly reflect what's persisted to disk. The issue is + that the local checkpoint is adapted as soon as an operation is processed (but not fsynced yet). This + leaves room for the history below the global checkpoint to still change in case of a crash. As we rely + on global checkpoints for CCR as well as operation-based recoveries, this has the risk of shard + copies / follower clusters going out of sync. + + This commit required changing some core classes in the system: + + - The LocalCheckpointTracker keeps track now not only of the information whether an operation has + been processed, but also whether that operation has been persisted to disk. + - TranslogWriter now keeps track of the sequence numbers that have not been fsynced yet. Once + they are fsynced, TranslogWriter notifies LocalCheckpointTracker of this. + - ReplicationTracker now keeps track of the persisted local and persisted global checkpoints of all + shard copies when in primary mode. The computed global checkpoint (which represents the + minimum of all persisted local checkpoints of all in-sync shard copies), which was previously stored + in the checkpoint entry for the local shard copy, has been moved to an extra field. + - The periodic global checkpoint sync now also takes async durability into account, where the local + checkpoints on shards only advance when the translog is asynchronously fsynced. This means that + the previous condition to detect inactivity (max sequence number is equal to global checkpoint) is + not sufficient anymore. + - The new index closing API does not work when combined with async durability. The shard + verification step is now requires an additional pre-flight step to fsync the translog, so that the main + verify shard step has the most up-to-date global checkpoint at disposition. + +commit adab7eae71f7ba72d571ae9f2174b3fdcd3afca2 +Author: Christoph Büscher +Date: Thu Jun 20 10:33:48 2019 +0200 + + [Docs] Remove boost parameter from intervals-query example (#43331) + + The boost factor doesn't seem to be needed and can be removed. + +commit 24cfca53fa71dc63700d6e83336095b28a0e9c72 +Author: Tanguy Leroux +Date: Thu Jun 20 10:06:14 2019 +0200 + + Reconnect remote cluster when seeds are changed (#43379) + + The RemoteClusterService should close the current + RemoteClusterConnection and should build it again if + the seeds are changed, similarly to what is done when + the ping interval or the compression settings are changed. + + Closes #37799 + +commit d684119618afa589d09e75b55ab9e7c6af730793 +Author: Andrei Stefan +Date: Thu Jun 20 10:15:43 2019 +0300 + + Remove mentions of "fields with the same name in the same index" (#43077) + + Together with types removal, any mention of "fields with the same name in the same index" doesn't make sense anymore. + + (cherry picked from commit c5190106cbd4c007945156249cce462956933326) + +commit fe0f9055d8de4626b2eb83ece4b63cdcadfdad75 +Author: Andrei Stefan +Date: Thu Jun 20 10:11:13 2019 +0300 + + Fix NPE in case of subsequent scrolled requests for a CSV/TSV formatted response (#43365) + + (cherry picked from commit 0ef7bb0f8b07cd0392d37f96ca9360821b19315a) + +commit 94a4bc99332822223ce6af0b9af459c75441e536 +Author: Luca Cavanna +Date: Wed Jun 19 11:18:40 2019 +0200 + + SearchPhaseContext to not extend ActionListener (#43269) + + The fact that SearchPhaseContext extends ActionListener makes it hard + to reason about when the original listener is notified and to trace + those calls. Also, the corresponding onFailure and onResponse were + only needed in two places, one each, where they can be replaced by a + more intuitive call, like sendSearchResponse for onResponse. + +commit 9de4e878f74fd711061b8d0fa0de7b2f96b862f5 +Merge: c8e6474eef0 c33d62adbc2 +Author: Martijn van Groningen +Date: Thu Jun 20 09:44:31 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit c33d62adbc2be5a019de1f2b5344e0ba1eb98d9a +Author: Jim Ferenczi +Date: Wed Jun 19 22:44:22 2019 +0200 + + Reduce the number of docvalues iterator created in the global ordinals fielddata (#43091) + + Today the fielddata for global ordinals re-creates docvalues readers of each segment + when building the iterator of a single segment. This is required because the lookup of + global ordinals needs to access the docvalues's TermsEnum of each segment to retrieve + the original terms. This also means that we need to create NxN (where N is the number of segment in the index) docvalues iterators + each time we want to collect global ordinal values. This wasn't an issue in previous versions since docvalues readers are stateless + before 6.0 so they are reused on each segment but now that docvalues are iterators we need to create a new instance each time + we want to access the values. In order to avoid creating too many iterators this change splits + the global ordinals fielddata in two classes, one that is used to cache a single instance per directory reader and one + that is created from the cached instance that can be used by a single consumer. The latter creates the TermsEnum of each segment + once and reuse them to create the segment's iterator. This prevents the creation of all TermsEnums each time we want to access + the value of a single segment, hence reducing the number of docvalues iterator to create to Nx2 (one iterator and one lookup per segment). + +commit 1f1a035def1a3d2db99133583b346c837a07e0b2 +Author: Jason Tedor +Date: Wed Jun 19 22:43:58 2019 -0400 + + Remove stale test logging annotations (#43403) + + This commit removes some very old test logging annotations that appeared + to be added to investigate test failures that are long since closed. If + these are needed, they can be added back on a case-by-case basis with a + comment associating them to a test failure. + +commit c2bf628a6df39e4b20bba12b7d172d850146ff6a +Author: Lee Hinman +Date: Wed Jun 19 16:37:41 2019 -0600 + + [7.x] Narrow period of Shrink action in which ILM prevents stopping (#43254) (#43393) + + * Narrow period of Shrink action in which ILM prevents stopping + + Prior to this change, we would prevent stopping of ILM if the index was + anywhere in the shrink action. This commit changes + `IndexLifecycleService` to allow stopping when in any of the innocuous + steps during shrink. This changes ILM only to prevent stopping if + absolutely necessary. + + Resolves #43253 + + * Rename variable for ignore actions -> ignore steps + + * Fix comment + + * Factor test out to test *all* stoppable steps + +commit 6b084e55c57c9ab8d7ab1725228b68d5db0f015b +Author: Lee Hinman +Date: Wed Jun 19 16:07:28 2019 -0600 + + [7.x] Prevent NullPointerException in TransportRolloverAction (#43353) (#43397) + + It's possible for the passed in `IndexMetaData` to be null (for + instance, cluster state passed in does not have the index in its + metadata) which in turn can cause a `NullPointerException` when + evaluating the conditions for an index. This commit adds null protection + and unit tests for this case. + + Resolves #43296 + +commit 77ce3260dd6f582c9dae5db543ac609fc59dd785 +Author: Benjamin Trent +Date: Wed Jun 19 16:19:06 2019 -0500 + + [ML][Data Frame] make response.count be total count of hits (#43241) (#43389) + + * [ML][Data Frame] make response.count be total count of hits + + * addressing line length check + + * changing response count for filters + + * adjusting serialization, variable name, and total count logic + + * making count mandatory for creation + +commit b333ced5a7eec3cca61695a371185710f9560947 +Author: Benjamin Trent +Date: Wed Jun 19 16:18:27 2019 -0500 + + [7.x] [ML][Data Frame] adds new pipeline field to dest config (#43124) (#43388) + + * [ML][Data Frame] adds new pipeline field to dest config (#43124) + + * [ML][Data Frame] adds new pipeline field to dest config + + * Adding pipeline support to _preview + + * removing unused import + + * moving towards extracting _source from pipeline simulation + + * fixing permission requirement, adding _index entry to doc + + * adjusting for java 8 compatibility + + * adjusting bwc serialization version to 7.3.0 + +commit b957aa46ce6bed05f282e7bab07812d7afa41568 +Author: Jim Ferenczi +Date: Wed Jun 19 21:25:21 2019 +0200 + + Allocate memory lazily in BestBucketsDeferringCollector (#43339) + + While investigating memory consumption of deeply nested aggregations for #43091 + the memory used to keep track of the doc ids and buckets in the BestBucketsDeferringCollector + showed up as one of the main contributor. In my tests half of the memory held in the + BestBucketsDeferringCollector is associated to segments that don't have matching docs + in the selected buckets. This is expected on fields that have a big cardinality since each + bucket can appear in very few segments. By allocating the builders lazily this change + reduces the memory consumption by a factor 2 (from 1GB to 512MB), hence reducing the + impact on gcs for these volatile allocations. This commit also switches the PackedLongValues.Builder + with a RoaringDocIdSet in order to handle very sparse buckets more efficiently. + + I ran all my tests on the `geoname` rally track with the following query: + + ```` + { + "size": 0, + "aggs": { + "country_population": { + "terms": { + "size": 100, + "field": "country_code.raw" + }, + "aggs": { + "admin1_code": { + "terms": { + "size": 100, + "field": "admin1_code.raw" + }, + "aggs": { + "admin2_code": { + "terms": { + "size": 100, + "field": "admin2_code.raw" + }, + "aggs": { + "sum_population": { + "sum": { + "field": "population" + } + } + } + } + } + } + } + } + } + } + ```` + +commit d1637ca476df6d1d3fa774eadbbce960363e626b +Author: Christos Soulios +Date: Wed Jun 19 22:31:06 2019 +0300 + + Backport: Refactor aggregation base classes to remove doEquals() and doHashCode() (#43363) + + This PR is a backport a of #43214 from v8.0.0 + + A number of the aggregation base classes have an abstract doEquals() and doHashCode() (e.g. InternalAggregation.java, AbstractPipelineAggregationBuilder.java). + + Theoretically this is so the sub-classes can add to the equals/hashCode and don't need to worry about calling super.equals(). In practice, it's mostly just confusing/inconsistent. And if there are more than two levels, we end up with situations like InternalMappedSignificantTerms which has to call super.doEquals() which defeats the point of having these overridable methods. + + This PR removes the do versions and just use equals/hashCode ensuring the super when necessary. + +commit be42b2c70cc99bdaca5aabd3fb400b5ca2450b3f +Author: Armin Braun +Date: Wed Jun 19 21:23:09 2019 +0200 + + Fix NetworkUtilsTests (#43295) (#43378) + + * Follow up to #42109: + * Adjust test to only check that interface lookup by name works not actually lookup IPs which is brittle since virtual interfaces can be destroyed/created by Docker while the tests are running + + Co-authored-by: Jason Tedor + +commit 1dde6ba1db91132095b71bd9efa9cdf447474e5d +Author: James Baiera +Date: Wed Jun 19 13:57:25 2019 -0400 + + Muting DataFrameTransformIT.testGetStats + + See #43324 + +commit bf74d38782b9cb8d19570da05df89127b2c04cb0 +Author: Jason Tedor +Date: Wed Jun 19 13:25:02 2019 -0400 + + Fix GeoIP custom database directory in docs (#43383) + + These docs were misleading for package installations of + Elasticsearch. Instead, we should refer to $ES_CONFIG/ingest-geoip as + the path to place the custom database files. For non-package + installations, this is the same as $ES_HOME/config, but for package + installations this is not the case as the config directory for package + installations is /etc/elasticsearch, and is not relative to + $ES_HOME. This commit corrects the docs. + +commit d81ce9a647cf209830fb7f6eaab70c0343a16c1a +Author: Lee Hinman +Date: Wed Jun 19 10:35:17 2019 -0600 + + Return 0 for negative "free" and "total" memory reported by the OS (#42725) + + * Return 0 for negative "free" and "total" memory reported by the OS + + We've had a situation where the MX bean reported negative values for the + free memory of the OS, in those rare cases we want to return a value of + 0 rather than blowing up later down the pipeline. + + In the event that there is a serialization or creation error with regard + to memory use, this adds asserts so the failure will occur as soon as + possible and give us a better location for investigation. + + Resolves #42157 + + * Fix test passing in invalid memory value + + * Fix another test passing in invalid memory value + + * Also change mem check in MachineLearning.machineMemoryFromStats + + * Add background documentation for why we prevent negative return values + + * Clarify comment a bit more + +commit 23a34713943d10f4d401f4f70275bdcb5657d788 +Author: Gordon Brown +Date: Wed Jun 19 10:34:39 2019 -0600 + + Fix randomization in testPerformActionAttrsRequestFails (#43304) + + The randomization in this test would occasionally generate duplicate + node attribute keys, causing spurious test failures. This commit adjusts + the randomization to not generate duplicate keys and cleans up the data + structure used to hold the generated keys. + +commit 667bdcd3ce6a7bed10e3e5bb166e9418877a8498 +Author: Armin Braun +Date: Wed Jun 19 18:03:39 2019 +0200 + + Ensure Test Cluster in Azure Discovery Tests Closes (#43057) (#43372) + + * Use `internalCluster().close()` to force all nodes (and not just the datanodes) to shut down even if one fails to shut down in time + * Force closing httpServer to get cleaner logs if nodes still hang on shut down + * Relates #43048 + +commit b5c8b32cab331eff70905f3fe8c9ca6faea7c6b9 +Author: Nhat Nguyen +Date: Wed Jun 19 09:56:57 2019 -0400 + + Do not use soft-deletes to resolve indexing strategy (#43336) + + This PR reverts #35230. + + Previously, we reply on soft-deletes to fill the mismatch between the + version map and the Lucene index. This is no longer needed after #43202 + where we rebuild the version map when opening an engine. Moreover, + PrunePostingsMergePolicy can prune _id of soft-deleted documents out of + order; thus the lookup result including soft-deletes sometimes does not + return the latest version (although it's okay as we only use a valid + result in an engine). + + With this change, we use only live documents in Lucene to resolve the + indexing strategy. This is perfectly safe since we keep all deleted + documents after the local checkpoint in the version map. + + Closes #42979 + +commit a4c45b5d70a146b90b844ec2d57beea8968b33d9 +Author: Martijn van Groningen +Date: Wed Jun 19 16:15:09 2019 +0200 + + Replace Streamable w/ Writeable in SingleShardRequest and subclasses (#43222) (#43364) + + Backport of: https://github.com/elastic/elasticsearch/pull/43222 + + This commit replaces usages of Streamable with Writeable for the + SingleShardRequest / TransportSingleShardAction classes and subclasses of + these classes. + + Note that where possible response fields were made final and default + constructors were removed. + + Relates to #34389 + +commit 86b58d9ff333396a4f69daeea520458471756680 +Author: Przemysław Witek +Date: Wed Jun 19 15:31:26 2019 +0200 + + Rename AutoDetectResultProcessor* to AutodetectResultProcessor* for consistency with other classes where the spelling is "Autodetect" (#43359) (#43366) + +commit 8578aba654d1350a925cb2f864d3d18632e03ee5 +Author: Paul Sanwald +Date: Wed Jun 19 07:06:45 2019 -0400 + + [backport] Adds a minimum interval to `auto_date_histogram`. (#42814) (#43285) + + Backports minimum interval to date histogram + +commit 2f173402eccd59e7ebddbc2eb9f7fe71cb38aab8 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Jun 19 18:26:52 2019 +1000 + + Add kerberos grant_type to get token in exchange for Kerberos ticket (#42847) (#43355) + + Kibana wants to create access_token/refresh_token pair using Token + management APIs in exchange for kerberos tickets. `client_credentials` + grant_type requires every user to have `cluster:admin/xpack/security/token/create` + cluster privilege. + + This commit introduces `_kerberos` grant_type for generating `access_token` + and `refresh_token` in exchange for a valid base64 encoded kerberos ticket. + In addition, `kibana_user` role now has cluster privilege to create tokens. + This allows Kibana to create access_token/refresh_token pair in exchange for + kerberos tickets. + + Note: + The lifetime from the kerberos ticket is not used in ES and so even after it expires + the access_token/refresh_token pair will be valid. Care must be taken to invalidate + such tokens using token management APIs if required. + + Closes #41943 + +commit 42cc27e74f57fee41b8c49f6e1320ba2c857999c +Author: Jason Tedor +Date: Tue Jun 18 22:31:09 2019 -0400 + + Remove token service trace logging in tests + + This commit removes some trace logging for the token service in the + rolling upgrade tests. If there is an active investigation here, it + would be best to annotate this line with a comment in the source + indicating such. From my digging, it does not appear there is an active + investigation that relies on this logging, so we remove it. + +commit fa0911308022cc6ba85fee70a75f398f8ef7caff +Author: Jason Tedor +Date: Tue Jun 18 22:26:51 2019 -0400 + + Remove trace logging for ML native multi-node tests + + This trace logging looks like it was copy/pasted from another test, + where the logging in that test was only added to investigate a test + failure. This commit removes the trace logging. + +commit b98574240dfa33d8b86037c1e82b2ea8ce462586 +Author: Jason Tedor +Date: Tue Jun 18 22:22:57 2019 -0400 + + Remove trace logging from ML datafeeds in tests + + This was added to investigate a test failure over two years ago, yet + left behind. Since the test failure has been addressed since then, this + commit removes the trace logging. + +commit 9f7d1ff2de9ba17044bad3bf57c373e1146d3fd1 +Author: Igor Motov +Date: Tue Jun 18 07:03:45 2019 -0700 + + Geo: Add coerce support to libs/geo WKT parser (#43273) + + Adds support for coercing not closed polygons and ignoring Z value + to libs/geo WKT parser. + + Closes #43173 + +commit de1a685ccee3cb72221d0c4a4545d79ed46e9263 +Author: Jim Ferenczi +Date: Tue Jun 18 20:17:28 2019 +0200 + + Fix sporadic failures in QueryStringQueryTests#testToQueryFuzzyQueryAutoFuziness (#43322) + + This commit ensures that the test does not use reserved keyword (OR, AND, NOT) + when generating the random query strings. + + Closes #43318 + +commit 0867ea75b7a64ef4019dc3e62b01ff2f7a6079ba +Author: Mark Vieira +Date: Tue Jun 18 09:17:14 2019 -0700 + + Properly format reproduction lines for test methods that contain periods (#43255) + +commit aa6248d4d72c568e88e9cd2bb51bf7d78014872c +Author: Mayya Sharipova +Date: Tue Jun 18 11:56:04 2019 -0400 + + Move dense_vector and sparse_vector to module (#43280) (#43333) + +commit 0a79bf431a466e8ed02bdb78a8d220923590d216 +Author: Ryan Ernst +Date: Tue Jun 18 07:23:05 2019 -0700 + + Deprecate native code info in xpack info api (#43297) + + The xpack info api currently returns native code info within each + feature. This commit deprecates retrieving that info, which is now + available directly in the ML info api. + +commit 11ef5e63aeca93a285af7e62d8600fcf8a9dd473 +Author: caminsha <31421093+caminsha@users.noreply.github.com> +Date: Tue Jun 18 15:51:00 2019 +0200 + + [DOCS] Added a new use case for transport.port (#42126) + +commit 459d57f4c5128c7f991231bcf0ac0c5037da9484 +Author: Przemysław Witek +Date: Tue Jun 18 15:32:34 2019 +0200 + + [7.x] [ML] BWC tests for job_stats.timing_stats field (#43267) (#43293) + +commit 818a7093776e42d42d9501d7f7d425c4c9e954ee +Author: Colin Goodheart-Smithe +Date: Tue Jun 18 13:27:29 2019 +0100 + + Fixes formatting of CCS compatibility table (#43231) + +commit 90a8589294ce2882b0c45234b22b6cf49a03384c +Author: David Turner +Date: Tue Jun 18 12:22:55 2019 +0100 + + Local node is discovered when cluster fails (#43316) + + Today the `ClusterFormationFailureHelper` does not include the local node in + the list of nodes it claims to have discovered. This means that it sometimes + reports that it has not discovered a quorum when in fact it has. This commit + adds the local node to the set of discovered nodes. + +commit 2e064e0d13241114c3418b031d20943d99e85bbf +Author: David Turner +Date: Tue Jun 18 12:10:01 2019 +0100 + + Allow election of nodes outside voting config (#43243) + + Today we suppress election attempts on master-eligible nodes that are not in + the voting configuration. In fact this restriction is not necessary: any + master-eligible node can safely become master as long as it has a fresh enough + cluster state and can gather a quorum of votes. Moreover, this restriction is + sometimes undesirable: there may be a reason why we do not want any of the + nodes in the voting configuration to become master. + + The reason for this restriction is as follows. If you want to shut the master + down then you might first exclude it from the voting configuration. When this + exclusion succeeds you might reasonably expect that a new master has been + elected, since the voting config exclusion is almost always a step towards + shutting the node down. If we allow nodes outside the voting configuration to + be the master then the excluded node will continue to be master, which is + confusing. + + This commit adjusts the logic to allow master-eligible nodes to attempt an + election even if they are not in the voting configuration. If such a master is + successfully elected then it adds itself to the voting configuration. This + commit also adjusts the logic that causes master nodes to abdicate when they + are excluded from the voting configuration, to avoid the confusion described + above. + + Relates #37712, #37802. + +commit 5a9c48369b6c8d1f9952454897a743241edb2f39 +Author: Alpar Torok +Date: Tue Jun 18 11:51:20 2019 +0300 + + TestClusters: Convert the security plugin (#43242) + + * TestClusters: Convert the security plugin + + This PR moves security tests to use TestClusters. + The TLS test required support in testclusters itself, so the correct + wait condition is configgured based on the cluster settings. + + * PR review + +commit 94930d0e84c63b6a90cd851c0b353c4ee1af1948 +Author: Alpar Torok +Date: Tue Jun 18 10:10:05 2019 +0300 + + Testclusters: convert ml qa tests (#43229) + + * Testclusters: convert ml qa tests + + This PR converts the ML tests to use testclusters. + +commit da97325790b912b937457132f2265008832ffaed +Author: David Roberts +Date: Tue Jun 18 09:09:54 2019 +0100 + + [ML] Speed up persistent task rechecks in ML failover tests (#43291) + + The ML failover tests sometimes need to wait for jobs to be + assigned to new nodes following a node failure. They wait + 10 seconds for this to happen. However, if the node that + failed was the master node and a new master was elected then + this 10 seconds might not be long enough as a refresh of the + memory stats will delay job assignment. Once the memory + refresh completes the persistent task will be assigned when + the next cluster state update occurs or after the periodic + recheck interval, which defaults to 30 seconds. Rather than + increase the length of the wait for assignment to 31 seconds, + this change decreases the periodic recheck interval to 1 + second. + + Fixes #43289 + +commit 74813360aba33cfef31060b04933f72f9d9e6f84 +Author: Mark Vieira +Date: Wed May 29 15:34:52 2019 -0700 + + Fix inverted condition so we never cache rest integ tests + +commit 3204e0255c3e7f723d1ac7153b4ed662c7bd432d +Author: debadair +Date: Mon Jun 17 19:56:44 2019 -0700 + + [DOCS] Sewing SME says it should be "size 70" needle. + +commit e524e45aed9446d50e9c321962f289753a5961ce +Author: debadair +Date: Mon Jun 17 19:18:51 2019 -0700 + + [DOCS] Fix typo: extraneous {es} + +commit d0a4d23027c3c5d903d3d13defc9228b4ff21d5b +Author: Ryan Ernst +Date: Mon Jun 17 17:50:47 2019 -0700 + + Add elasticsearch distribution plugin (#43247) + + Several types of distributions are built and tested in elasticsearch, + ranging from the current version, to building or downloading snapshot or + released versions. Currently tests relying on these have to contain + logic deciding where and how to pull down these distributions. + + This commit adds an distributiond download plugin for each project to + manage which versions and variants the project needs. It abstracts away + all need for knowing where a particular version comes from, like a local + distribution or bwc project, or pulling from the elastic download + service. This will be used in a followup PR by the testclusters and + vagrant tests. + +commit 9767fc2c95fdd7a14daf5f65ee97b25c5ed38f10 +Author: debadair +Date: Mon Jun 17 16:57:43 2019 -0700 + + [DOCS] Add introduction to Elasticsearch. (#43075) + + * [DOCS] Add introduction to Elasticsearch. + + * [DOCS] Incorporated review comments. + + * [DOCS] Minor edits to add an abbreviated title and cross refs. + + * [DOCS] Added sizing tips & link to quantatative sizing video. + +commit 0c5086d2f36dc9119b668652d9a1cd29d9935c1f +Author: Nhat Nguyen +Date: Mon Jun 17 17:50:54 2019 -0400 + + Rebuild version map when opening internal engine (#43202) + + With this change, we will rebuild the live version map and local + checkpoint using documents (including soft-deleted) from the safe commit + when opening an internal engine. This allows us to safely prune away _id + of all soft-deleted documents as the version map is always in-sync with + the Lucene index. + + Relates #40741 + Supersedes #42979 + +commit 365f87c6227323f7f951e9a267442260e1f71d1f +Author: Benjamin Trent +Date: Mon Jun 17 16:49:00 2019 -0500 + + [ML][Data Frame] only complete task after state persistence (#43230) (#43294) + + * [ML][Data Frame] only complete task after state persistence + + There is a race condition where the task could be completed, but there + is still a pending document write. This change moves + the task cancellation into the actionlistener of the state persistence. + + intermediate commit + + intermediate commit + + * removing unused import + + * removing unused const + + * refreshing internal index after waiting for task to complete + + * adjusting test data generation + +commit 04a7c84e8ba45def3d832ddfadbd710229a7e32f +Author: Jack Conradson +Date: Mon Jun 17 10:58:53 2019 -0700 + + Add Painless Docs for Datetime Inputs (#43128) + + This changes add documentation for accessing datetimes in Painless scripts + from the three most common inputs of params, _source, and doc. + +commit 7ed23088c1091da6a80f2da7cf1aa0ca38648152 +Author: lcawl +Date: Mon Jun 17 10:08:08 2019 -0700 + + [DOCS] Fixes formatting of 7.2 breaking changes + +commit 2d9b3a69e8f71ab6545a09dcd9ad8a025c1ced3e +Author: David Turner +Date: Mon Jun 17 17:13:04 2019 +0100 + + Relocation targets are assigned shards too (#43276) + + Adds relocation targets to the output of + `IndexShardRoutingTable#assignedShards`. + +commit 8b1b9f8ab9074678fdfd96096a124caa4ac825cf +Author: Martijn Laarman +Date: Mon Jun 17 16:57:13 2019 +0200 + + Introduce stability description to the REST API specification (#38413) (#43278) + + * introduce state to the REST API specification + + * change state over to stability + + * CCR is no GA updated to stable + + * SQL is now GA so marked as stable + + * Introduce `internal` as state for API's, marks stable in terms of lifetime but unstable in terms of guarantees on its output format since it exposes internal representations + + * make setting a wrong stability value, or not setting it at all an error that causes the YAML test suite to fail + + * update spec files to be explicit about their stability state + + * Document the fact that stability needs to be defined + + Otherwise the YAML test runner will fail (with a nice exception message) + + * address check style violations + + * update rest spec unit tests to include stability + + * found one more test spec file not declaring stability, made sure stability appears after documentation everywhere + + * cluster.state is stable, mark response in some way to denote its a key value format that can be changed during minors + + * mark data frame API's as beta + + * remove internal and private as states for an API + + * removed the wrong enum values in the Stability Enum in the previous commit + + (cherry picked from commit 61c34bbd92f8f7e5f22fa411c6b682b0ebd8a99d) + +commit 21da84edbcbe03f0efaa02f9ae3f2c4f6d856e76 +Author: Lee Hinman +Date: Mon Jun 17 08:44:49 2019 -0600 + + Make ILM force merging best effort (#43246) + + It's possible for force merges kicked off by ILM to silently stop (due + to a node relocating for example). In which case, the segment count may + not reach what the user configured. In the subsequent `SegmentCountStep` + waiting for the expected segment count may wait indefinitely. Because of + this, this commit makes force merges "best effort" and then changes the + `SegmentCountStep` to simply report (at INFO level) if the merge was not + successful. + + Relates to #42824 + Resolves #43245 + +commit ba15d08e148dce680277677247ac4be8fe5f09d1 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jun 17 15:04:17 2019 +0200 + + Allow cluster access during node restart (#42946) (#43272) + + This commit modifies InternalTestCluster to allow using client() and + other operations inside a RestartCallback (onStoppedNode typically). + Restarting nodes are now removed from the map and thus all + methods now return the state as if the restarting node does not exist. + + This avoids various exceptions stemming from accessing the stopped + node(s). + +commit 4b58827bebf984dc9a6c537ddb9405bcf3333268 +Author: David Turner +Date: Mon Jun 17 10:22:11 2019 +0100 + + Make DiscoveryNodeRole into a value object (#43257) + + Adds `equals()` and `hashcode()` methods to `DiscoveryNodeRole` to compare + these objects' values for equality, and adds a field to allow us to distinguish + unknown roles from known ones with the same name and abbreviation, for clearer + test failures. + + Relates #43175 + +commit 3effe264dac57982661cf80ad5052f0547ae7306 +Author: David Roberts +Date: Mon Jun 17 09:26:36 2019 +0100 + + [ML] Fix problem with lost shards in distributed failure test (#43153) + + We were stopping a node in the cluster at a time when + the replica shards of the .ml-state index might not + have been created. This change moves the wait for + green status to a point where the .ml-state index + exists. + + Fixes #40546 + Fixes #41742 + + Forward port of #43111 + +commit a8bf18184ac85b1936a821bec25e28643d9f76cf +Author: Alpar Torok +Date: Mon Jun 17 10:49:20 2019 +0300 + + Refactor Version class to make version bumps easier (#42668) (#43215) + + With this change we only have to add one line to add a new version. + The intent is to make it less error prone and easier to write a script + to automate the process. + +commit e9e8243faa037d8a695ad3be07a37b7253f7d24b +Author: István Zoltán Szabó +Date: Mon Jun 17 09:33:57 2019 +0200 + + [DOCS] Simplifies wording. (#43226) + + This PR simplifies the wording of the TOC and eventually makes it shorter. + +commit b2613a123d90c194189314d2f0e18d218c084797 +Author: Przemysław Witek +Date: Mon Jun 17 08:58:26 2019 +0200 + + [7.x] Report exponential_avg_bucket_processing_time which gives more weight to recent buckets (#43189) (#43263) + +commit a191ebabba65a0e1d0433bd0e59c5ab508bca3d6 +Author: Alpar Torok +Date: Mon Jun 17 09:24:18 2019 +0300 + + TestClusters: convert kerberos-tests (#43232) + + Looks like cluster formation tasks no longer plays nice wit + test.fixtures so we just convert this to use testclusters. + +commit 4b643c50fad182580677fe8bc7155257ebf00267 +Author: Nhat Nguyen +Date: Fri Jun 14 12:43:44 2019 -0400 + + Account soft deletes in committed segments (#43126) + + This change fixes the delete count issue in segment stats where we don't + account soft-deleted documents from committed segments. + + Relates #43103 + +commit c3f1e6a54230a11696df29d286e208a4b0034b5f +Author: Jay Modi +Date: Fri Jun 14 12:29:43 2019 -0600 + + Ensure threads running before closing node (#43240) + + There are a few tests within NodeTests that submit items to the + threadpool and then close the node. The tests are designed to check + how running tasks are affected during node close. These tests can cause + CI failures since the submitted tasks may not be running when the node + is closed and then execute after the thread context is closed, which + triggers an unexpected exception. This change ensures the threads are + running so we avoid the unexpected exception and can test these cases. + + The test of task submittal while a node is closing is also important so + an additional but muted test has been added that tests the case where a + task may be getting submitted while the node is closing and ensuring we + do not trigger anything unexpected in these cases. + + Relates #42774 + Relates #42577 + +commit 4b1d8e443377fb9effbb8e1cff2889765332842a +Author: Julie Tibshirani +Date: Thu Jun 13 12:07:40 2019 -0700 + + Allow big integers and decimals to be mapped dynamically. (#42827) + + This PR proposes to model big integers as longs (and big decimals as doubles) + in the context of dynamic mappings. + + Previously, the dynamic mapping logic did not recognize big integers or + decimals, and would an error of the form "No matching token for number_type + [BIG_INTEGER]" when a dynamic big integer was encountered. It now accepts these + numeric types and interprets them as 'long' and 'double' respectively. This + allows `dynamic_templates` to accept and and remap them as another type such as + `keyword` or `scaled_float`. + + Addresses #37846. + +commit 3928c624a326da7b6a66462f19163e21088f7920 +Author: David Roberts +Date: Fri Jun 14 17:53:55 2019 +0100 + + [ML] Close sample stream in post_data endpoint (#43235) + + A static code analysis revealed that we are not closing + the input stream in the post_data endpoint. This + actually makes no difference in practice, as the + particular InputStream implementation in this case is + org.elasticsearch.common.bytes.BytesReferenceStreamInput + and its close() method is a no-op. However, it is good + practice to close the stream anyway. + +commit a7bdea8a15d7fc963fe2d0d62cb598d1b10e155b +Author: Mayya Sharipova +Date: Fri Jun 14 12:41:27 2019 -0400 + + BWC tests - move vector distance functions to 7.3 + +commit 982a23f8c3acdc9c88694ae2d5393da9c28c21c0 +Author: Lisa Cawley +Date: Fri Jun 14 08:42:18 2019 -0700 + + [DOCS] Adds size and from parameters to data frame APIs (#43212) + +commit 8c66149e2dea4505c7b06eac387b4787049a4ee3 +Author: Benjamin Trent +Date: Fri Jun 14 10:43:36 2019 -0500 + + [ML][Data Frame] have sum map to a double to prevent overflows (#43213) (#43219) + +commit 9cd89c3453d50b28949179f5a3afde7cb735bf4e +Author: Marios Trivyzas +Date: Fri Jun 14 13:26:18 2019 +0200 + + SQL: Increase hard limit for sorting on aggregates (#43220) + + To be consistent with the `search.max_buckets` default setting, + set the hard limit of the PriorityQueue used for in memory sorting, + when sorting on an aggregate function, to 10000. + + Fixes: #43168 + + (cherry picked from commit 079e012fdea68ea0a7daae078359495047e9c407) + +commit be9f27bb1685dde9f13b21b30b7a81eae315b991 +Author: Yannick Welsch +Date: Fri Jun 14 13:31:10 2019 +0200 + + Properly use cancellable threads to stop UnicastZenPing (#42844) + + Fixes a backport issue with #42884 where Zen1 was not properly taken into account. + +commit 221d23de9f8295864ed8c4553190c4312ca6c81b +Author: David Turner +Date: Fri Jun 14 12:27:01 2019 +0100 + + Fix DiscoveryNodeRoleIT (#43225) + + The test fails if querying the roles via a transport client, since the + transport client does not have the plugin necessary to interpret the additional + role correctly. This commit adds this plugin to the transport client used. + + Relates #43175 + Fixes #43223 + +commit 7af23324e3291176bce59535ead95d59de326f4f +Author: Christoph Büscher +Date: Fri Jun 14 11:14:53 2019 +0200 + + SimpleQ.S.B and QueryStringQ.S.B tests should avoid `now` in query (#43199) + + Currently the randomization of the q.b. in these tests can create query strings + that can cause caching to be disabled for this query if we query all fields and + there is a date field present. This is pretty much an anomaly that we shouldn't + generally test for in the "testToQuery" tests where cache policies are checked. + + This change makes sure we don't create offending query strings so the cache + checks never hit these cases and adds a special test method to check this edge + case. + + Closes #43112 + +commit 4c8e77e092ea0790576e57f7cbd7ce77b874b5ad +Author: Przemyslaw Gomulka +Date: Fri Jun 14 10:57:22 2019 +0200 + + Disable DiscoveryNodeRoleIT test due to failures (#43224) + + relates #43223 + +commit cce5b0f0181fb01bea0a5d74d5e32748288f2c0a +Author: Alpar Torok +Date: Fri Jun 14 10:57:37 2019 +0300 + + Convert dataframes to use testclusters (#43032) + +commit 023a126b5726a8e2beb951a2d7459e43cab2952d +Author: Alpar Torok +Date: Fri Jun 14 10:51:03 2019 +0300 + + Correctly configure testclsuters for fips (#43186) + + Configuration was not being triggered for all projects with the way it + was previusly set up. + With this change we do it for each project directly. + +commit 7cc6dca69713e412d9fbb9317538cccb3157e926 +Author: Alpar Torok +Date: Fri Jun 14 10:35:21 2019 +0300 + + Remove explicily enabled build fixture task + +commit 65a584b6fb79be88fc3480ddb3528a74f398158e +Author: Przemysław Witek +Date: Fri Jun 14 09:03:14 2019 +0200 + + [7.x] Report timing stats as part of the Job stats response (#42709) (#43193) + +commit d27c0fd50d56a8d6b41dc76ec472a690ffee1fef +Author: Przemyslaw Gomulka +Date: Fri Jun 14 08:56:26 2019 +0200 + + Fix roundUp parsing with composite patterns backport(#43080) (#43191) + + roundUp parsers were losing the composite pattern information when new + JavaDateFormatter was created from methods withLocale or withZone. + + The roundUp parser should be preserved when calling these methods. This is the same approach in withLocale/Zone methods as in https://github.com/elastic/elasticsearch/blob/daa2ec8a605d385a65b9ab3e89d016b3fd0dffe2/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java + + closes #42835 + +commit 41ac1cd16c6056c5fd4c19133fb53ccc9633d75c +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jun 14 08:16:11 2019 +0200 + + Fix versions for reindex max_docs tests (#42951) + + Versions were assuming this went into 7.2, but this ended up going into + 7.3, updated test versions to reflect that. + + Related to #41894 + +commit 8a341a3ea5093d336a37174f7f169374d7a703a3 +Author: lcawl +Date: Thu Jun 13 11:15:24 2019 -0700 + + [DOCS] Fix link to ML node description + +commit 7b90ceae0cf26bc1c4a1da0bbdb3756375732cb4 +Author: Lisa Cawley +Date: Thu Jun 13 09:46:55 2019 -0700 + + [DOCS] Update node descriptions for default distribution (#42812) + +commit 2bcc49424d6c0dba1a5bead2fa8eeff30d96d48e +Author: Jason Tedor +Date: Thu Jun 13 16:45:48 2019 -0400 + + Register possible node roles in transport client + + The transport client needs to be told about the possible node + roles. This commit does that. + +commit 3c73602524aa9a160e2ec7e263bc97095f4dd411 +Author: Marios Trivyzas +Date: Thu Jun 13 21:21:32 2019 +0200 + + SQL: Fix wrong results when sorting on aggregate (#43154) + + - Previously, when shorting on an aggregate function the bucket + processing ended early when the explicit (LIMIT XXX) or the impliciti + limit of 512 was reached. As a consequence, only a set of grouping + buckets was processed and the results returned didn't reflect the global + ordering. + + - Previously, the priority queue shorting method had an inverse + comparison check and the final response from the priority queue was also + returned in the inversed order because of the calls to the `pop()` + method. + + Fixes: #42851 + + (cherry picked from commit 19909edcfdf5792b38c1363b07379783ebd0e6c4) + +commit 55dba6ffade328b7cf795fb50e7322ce81e9ad09 +Author: Jason Tedor +Date: Thu Jun 13 15:44:40 2019 -0400 + + Fix JDK-version dependent exception message parsing + + This commit fixes some JDK-version dependent exception message checking + in the discovery node role tests. + +commit 5bc3b7f741a4c60e4ef4d81314951532654bf8ee +Author: Jason Tedor +Date: Thu Jun 13 14:43:14 2019 -0400 + + Enable node roles to be pluggable (#43175) + + This commit introduces the possibility for a plugin to introduce + additional node roles. + +commit c3ce3f68919a75f03895a9a94b6ab00de11056a8 +Author: Ryan Ernst +Date: Thu Jun 13 11:38:29 2019 -0700 + + Add native code info to ML info api (#43172) + + The machine learning feature of xpack has native binaries with a + different commit id than the rest of code. It is currently exposed in + the xpack info api. This commit adds that commit information to the ML + info api, so that it may be removed from the info api. + +commit 8f7cd844229e89432b33aaf067cb15186f0905e4 +Author: Przemyslaw Gomulka +Date: Thu Jun 13 20:19:17 2019 +0200 + + Disable x-pack:qa:kerberos-tests due to failures (#43208) + + relates #40678 + +commit 4ba94a5051e248775556a71fd476abf025958e22 +Author: Alpar Torok +Date: Wed May 29 16:16:04 2019 +0300 + + Testclusters: convert ccr tests (#42313) + +commit c8e6474eef0249b042d9f8d282a2a0163555efeb +Author: Martijn van Groningen +Date: Thu Jun 13 16:48:58 2019 +0200 + + Changes required for merging in 7.x branch. + +commit 1f3db7eb3eb3e988d6e5fca1f49af7c7475a866b +Merge: 9d56a0365f1 43665183c2d +Author: Martijn van Groningen +Date: Thu Jun 13 16:49:38 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 43665183c2d850f23237755449e9ca7c54965567 +Author: David Roberts +Date: Thu Jun 13 13:14:33 2019 +0100 + + [ML] Restrict detection of epoch timestamps in find_file_structure (#43188) + + Previously 10 digit numbers were considered candidates to be + timestamps recorded as seconds since the epoch and 13 digit + numbers as timestamps recorded as milliseconds since the epoch. + + However, this meant that we could detect these formats for + numbers that would represent times far in the future. As an + example ISBN numbers starting with 9 were detected as milliseconds + since the epoch since they had 13 digits. + + This change tweaks the logic for detecting such timestamps to + require that they begin with 1 or 2. This means that numbers + that would represent times beyond about 2065 are no longer + detected as epoch timestamps. (We can add 3 to the definition + as we get closer to the cutoff date.) + +commit 8b3716553a56c62813116358953fdf28ea450bb7 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jun 13 10:30:23 2019 +0200 + + Remote reindex failure parse fix (#42928) + + A search request that partially fails with failures without an index + (index: null) in the failure would cause a parse error in reindex from + remote. This would hide the original exception, making it hard to debug + the root cause. This commit fixes this so that we can tolerate null + index entries in a search failure. + +commit a28569462f3cb7d9d294a61e747ca9a5328e3464 +Author: Luca Cavanna +Date: Thu Jun 13 11:18:07 2019 +0200 + + Add 6.8 to the remote clusters compatibility table (#42389) + + The table does not include 6.8 as it was written before we knew we were releasing it. This commit adds it. + +commit 167e51335d1d35e6fb8d3a5edaf43ede74fdf938 +Author: Alpar Torok +Date: Thu Jun 13 12:18:55 2019 +0300 + + Convert ILM tests to use testclusters (#43076) + + Also improove the error message when bin scripts are not found + +commit 209a493b27b9d1d1cc452c8723491ccce2c74fd8 +Author: Vincent Boulaye +Date: Thu Jun 13 11:17:45 2019 +0200 + + convert EmptyDirTask.groovy to .java (#34672) + +commit 47d861c23a13f6c80d1c07adb01f8a3dc933789a +Author: Gabryel Monteiro +Date: Thu Jun 13 06:12:02 2019 -0300 + + Converting DependencyLicensesTask and UpdateShasTask to java (#41921) + +commit afbb79196979ff5e52fd44f2d8097b59a4534a57 +Author: Mirek Svoboda +Date: Thu Jun 13 10:01:47 2019 +0200 + + Document wildcard for network interfaces (#28839) + + With this commit we mention how Elasticsearch behaves when + either `0` or `0.0.0.0` is used for `network.host`. + +commit eb7a8bb4a46ba367ed04c0da3d3defc5effffabf +Author: Alpar Torok +Date: Thu Jun 13 09:46:55 2019 +0300 + + Testclusters: graph (#43033) + + Convert x-pack graph to use testClusters + +commit a1b8d48f17f76512e1fc7fef3f341be8d8015564 +Author: Rafiullah Hamedy +Date: Thu Jun 13 01:14:34 2019 -0400 + + Convert PluginPropertiesExtension Groovy to Java (#39605) + +commit f70141c8629c8c7527d0d92e4d2c9fc4dcacdfbc +Author: Simon Willnauer +Date: Thu Jun 13 07:53:27 2019 +0200 + + Only load FST off heap if we are actually using mmaps for the term dictionary (#43158) + + Given the significant performance impact that NIOFS has when term dicts are + loaded off-heap this change enforces FstLoadMode#AUTO that loads term dicts + off heap only if the underlying index input indicates a memory map. + + Relates to #43150 + +commit 4ae1e30a988b24c3e576ea41d53d90a9c6fc98a7 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Jun 13 13:02:16 2019 +1000 + + Enable krb5kdc-fixture, kerberos tests mount urandom for kdc container (#41710) (#43178) + + Infra has fixed #10462 by installing `haveged` on CI workers. + This commit enables the disabled fixture and tests, and mounts + `/dev/urandom` for the container so there is enough + entropy required for kdc. + Note: hdfs-repository tests have been disabled, will raise a separate issue for it. + + Closes #40624 Closes #40678 + +commit 20031fb13fbd2ae23fcacbc3861ed0cb12b13bd2 +Author: Tal Levy +Date: Wed Jun 12 18:17:23 2019 -0700 + + Introduce unit tests for ValuesSourceType (#43174) (#43176) + + As the ValuesSourceType evolves, it is important to be + confident that new enum constants do not break + backwards-compatibility on the stream. Having dedicated + unit tests for this class will help be sure of that. + +commit 7c9acdb0ac17850c0aad1d27d7ec38cca72e60e6 +Author: Lisa Cawley +Date: Wed Jun 12 13:43:53 2019 -0700 + + [DOCS] Adds ML release highlights (#43169) + +commit ec50d4d2816b9ef17b209b3adb83e4f32c36a985 +Author: Benjamin Trent +Date: Wed Jun 12 14:50:17 2019 -0500 + + [ML][Data Frame] write a warning audit on bulk index failures (#43106) (#43171) + + * [ML][Data Frame] write a warning audit on bulk index failures + + * adding failure message and moving to use volalitile + +commit aff47954410e035473b012eeb521854201d4807b +Author: Benjamin Trent +Date: Wed Jun 12 14:39:38 2019 -0500 + + [ML][Data Frame] cleaning up tests since tasks are cancelled onfinish (#43136) (#43166) + + * [ML][Data Frame] cleaning up usage test since tasks are cancelled onfinish + + * Update DataFrameUsageIT.java + + * Fixing additional test, waiting for task to complete + + * removing unused import + + * unmuting test + +commit 6cfed7ec7289e62bc50c1362b3bad1c85f456ff8 +Author: Jim Ferenczi +Date: Wed Jun 12 20:52:57 2019 +0200 + + Also mmap terms index (`.tip`) files for hybridfs (#43150) + + This change adds the terms index (`.tip`) to the list of extensions + that are memory-mapped by hybridfs. These files used to be accessed + only once to load the terms index on-heap but since #42838 they can + now be used to read the binary FST directly so it is benefical to + memory-map them instead of accessing them via NIO. + +commit 51618af056c2462dd6a2fdd0bbe4b79be2066f2e +Author: James Baiera +Date: Wed Jun 12 14:34:31 2019 -0400 + + shrink may full copy when using multi data paths (#42913) (#42961) + + Additional scenario for full segment copy if hard link + cannot work across disks. + +commit b110164bf4b6d08c6850bb10d6d0402dddd901a1 +Author: Benjamin Trent +Date: Wed Jun 12 13:22:46 2019 -0500 + + [ML][Data Frame] add the src priv check for view_index_metadata (#43118) (#43161) + +commit f13f55ede34bc4a0e900214796b5d6f24b29438e +Author: Benjamin Trent +Date: Wed Jun 12 13:22:34 2019 -0500 + + [ML][Data Frame] change failure count reset logic (#43064) (#43159) + +commit 8711a092bffc9e2a7b16a427cadefe1b89a05e1e +Author: Yannick Welsch +Date: Wed Jun 12 18:49:59 2019 +0200 + + Stop SeedHostsResolver on shutdown (#42844) + + Fixes an issue where tests would sometimes hang for 5 seconds when restarting a node. The reason + is that the SeedHostsResolver is blockingly waiting on a result for the full 5 seconds when the + corresponding threadpool is shut down. + +commit 9d2adfb41ea4767d783058e195d38aa5c20c1243 +Author: Simon Willnauer +Date: Wed Jun 12 17:42:08 2019 +0200 + + Remove usage of FileSwitchDirectory (#42937) + + We are still using `FileSwitchDirectory` in the case a user configures file based pre-load of mmaps. This is trappy for multiple reasons if the both directories used by `FileSwitchDirectory` point to the same filesystem directory. One issue is LUCENE-8835 that cause issues like #37111 - unless LUCENE-8835 isn't fixed we should not use it in elasticsearch. Instead we use a similar trick as we use for HybridFS and subclass mmap directory directly. + +commit 7f2f0b7620ff2f3d21a4e80ba5d8036323998765 +Author: Lisa Cawley +Date: Wed Jun 12 10:13:04 2019 -0700 + + [DOCS] Adds dataframe authorization details (#43009) + +commit 597ae5c7b8db4ab616429a820044be1280a099f4 +Author: David Kyle +Date: Wed Jun 12 17:56:47 2019 +0100 + + [ML DataFrame] Reject Data Frame Ids containing upper case characters (#43145) + +commit 9de1c69c284823618acdfacfb125ca82faa4fc1c +Author: Alan Woodward +Date: Wed Jun 12 17:47:56 2019 +0100 + + IndexAnalyzers doesn't need to extend AbstractIndexComponent (#43149) + + AIC doesn't add anything here, and it removes the need to pass index settings + to the constructor. + +commit b8b24ccd8e8f2323c361b734959c631dd9187317 +Author: Jay Modi +Date: Wed Jun 12 09:36:04 2019 -0600 + + Fix mustache search template test (#43120) + + This commit fixes a race in the test for the new response format with + search templates. The test indexes a document and then executes a + search with the expectation of 0 results. In some instances, the index + will refresh prior to the search execution and 1 hit will be found + causing the test fail. + + Closes #42664 + +commit 5be0fb32f8635d7dd88a7cba0a8f3a03bb225cc2 +Author: Ryan Ernst +Date: Wed Jun 12 08:19:14 2019 -0700 + + Move painless context api spec to test local (#43122) + + The painless context api is internal and currently meant only for use in + generating docs. This commit moves the spec file for the api so that it + is only used by the test for this api, and not externally by any clients + building from the public rest spec. + +commit 9d56a0365f19a564384b0d870590533df5808538 +Author: James Baiera +Date: Mon Jun 10 16:33:12 2019 -0400 + + Limit a enrich policy execution to only one at a time (#42535) + + Add a keyed lock mechanism to the policy executor to ensure that an enrich policy + can only have one execution happening at a time. + +commit 79614aeb2dfe3cae948d9ce5c5d200055833a866 +Author: Jim Ferenczi +Date: Wed Jun 12 15:37:05 2019 +0200 + + SearchRequest#allowPartialSearchResults does not handle successful retries (#43095) + + When set to false, allowPartialSearchResults option does not check if the + shard failures have been reseted to null. The atomic array, that is used to record + shard failures, is filled with a null value if a successful request on a shard happens + after a failure on a shard of another replica. In this case the atomic array is not empty + but contains only null values so this shouldn't be considered as a failure since all + shards are successful (some replicas have failed but the retries on another replica succeeded). + This change fixes this bug by checking the content of the atomic array and fails the request only + if allowPartialSearchResults is set to false and at least one shard failure is not null. + + Closes #40743 + +commit 7f690e860645236b867ceb0d438c9e1778d3a53d +Author: Christoph Büscher +Date: Wed Jun 12 15:41:41 2019 +0200 + + Fix suggestions for empty indices (#42927) + + Currently suggesters return null values on empty shards. Usually this gets replaced + by results from other non-epmty shards, but if the index is completely epmty (e.g. after + creation) the search responses "suggest" is also "null" and we don't render a corresponding + output in the REST response. This is an irritating edge case that requires special handling on + the user side (see #42473) and should be fixed. + + This change makes sure every suggester type (completion, terms, phrase) returns at least an + empty skeleton suggestion output, even for empty shards. This way, even if we don't find + any suggestions anywhere, we still return and output the empty suggestion. + + Closes #42473 + +commit 5ae2460782fc51548ea7a2ec1ea4794c0e131590 +Author: Shaunak Kashyap +Date: Wed Jun 12 06:25:30 2019 -0700 + + [7.x] Metricbeat monitoring Elasticsearch: Reorder/remove steps (#42917) (#43130) + +commit 110f0c5b7e821dbe8fdb275bbe6e4f6be2bd1a28 +Author: Yannick Welsch +Date: Wed Jun 12 14:11:12 2019 +0200 + + Mute testDataFrameTransformCrud + + Relates to #43139 + +commit 6f9503800158c0af077dbcdeac251d56403545e3 +Author: Alexander Reelsen +Date: Wed Jun 12 12:42:01 2019 +0200 + + Upgrade HPPC to version 0.8.1 (#43025) + +commit b28e006f7ce51790829da29c3f816161bdf163b2 +Author: Dimitris Athanasiou +Date: Wed Jun 12 14:07:17 2019 +0300 + + [ML] Lock down extraction method when possible (#43104) (#43140) + +commit e5a4a2272b82f8f6b71a402a2a12b88cca98e957 +Author: Yannick Welsch +Date: Fri May 24 17:18:53 2019 +0200 + + Wipe repositories more often (#42511) + + Fixes an issue where repositories are unintentionally shared among tests (given that the repo contents is captured in a static variable on the test class, to allow "sharing" among nodes) and two tests randomly chose the same snapshot name, leading to a conflict. + + Closes #42519 + +commit 4da0fadedc18c58c8ab2dca5c9eacce87994e67c +Author: Luca Cavanna +Date: Tue Jun 11 10:13:03 2019 +0200 + + [DOCS] Clarify phrase suggester docs smoothing parameter (#42947) + + Closes #28512 + +commit afeda1a7b9a13f31fa6bc465c003028f4b0f1146 +Author: Luca Cavanna +Date: Tue Jun 11 10:11:58 2019 +0200 + + Split search in two when made against throttled and non throttled searches (#42510) + + When a search on some indices takes a long time, it may cause problems to other indices that are being searched as part of the same search request and being written to as well, because their search context needs to stay open for a long time. This is especially a problem when searching against throttled and non-throttled indices as part of the same request. The problem can be generalized though: this may happen whenever read-only indices are searched together with indices that are being written to. Search contexts staying open for a long time is only an issue for indices that are being written to, in practice. + + This commit splits the search in two sub-searches: one for read-only indices, and one for ordinary indices. This way the two don't interfere with each other. The split is done only when size is greater than 0, no scroll is provided and query_then_fetch is used as search type. Otherwise, the search executes like before. Note that the returned num_reduce_phases reflect the number of reduction phases that were run. If the search is split in two, there are three reductions: one non-final for each search, and a final one that merges the results of the previous two. + + Closes #40900 + +commit e53859265260ae0fd6a18b7eb912605d7ff80c27 +Author: Luca Cavanna +Date: Thu May 23 16:02:46 2019 +0200 + + Update max_concurrent_shard_request parameter docs (#42227) + + Some of the docs were outdated as they did not mention that the limit is + not per node. Also, The default value changed. + + Relates to #31206 + +commit 31e8bff2acd89e201e5517e76a2faed3d5a3ff62 +Author: Luca Cavanna +Date: Thu May 23 11:52:39 2019 +0200 + + Rename SearchRequest#crossClusterSearch (#42363) + + The SearchRequest#crossClusterSearch method is currently used only as + part of cross cluster search request, when minimizing roundtrips. + It will soon be used also when splitting a search into two: one for + throttled and one for non throttled indices. It will probably be used + for other usecases as well in the future, hence it makes sense to generalize its name to subSearchRequest. + +commit a75964d8fd1f407409d6408ab3dc6c5dcfcdaeea +Author: markharwood +Date: Wed Jun 12 09:26:26 2019 +0100 + + Docs change for exists query. (#43092) + + Now emphasises the test is for indexed values. + Previous documentation only mentioned the state of the input JSON doc (null values) but this is only one of several reasons why an indexed value may not exist. + + Closes #24256 + +commit f7d4ca7a14feb8a17990f7a66a7d2e089ce588e5 +Author: Jack Conradson +Date: Tue Jun 11 13:15:02 2019 -0700 + + Exclude internal SQL methods from Painless API Docs (#43116) + +commit 5692be21619a67d2e40de4963395a25e3dc57fa7 +Author: Nhat Nguyen +Date: Tue Jun 11 17:55:32 2019 -0400 + + Fix timing issue in CcrRetentionLeaseIT (#43054) + + In these tests, we sleep for a small multiple of the renew interval, + then check that the retention leases are not changed. If a renewal + request takes longer than that interval because of GC or slow CI, then + the retention leases are not the same as before sleep. With this change, + we relax to assert that we eventually stop the renewable process. + + Closes #39509 + +commit 7ff3d86cf00d91dadd5cab4d0770e2fa1b27bff5 +Author: Benjamin Trent +Date: Tue Jun 11 15:55:18 2019 -0500 + + [ML][Data Frame] adding dest.index and id validations (#43053) (#43109) + + * [ML][Data Frame] adding dest.index and id validations + + * adjusting message format + + * Adjusting id validity pattern + + * Update DataFrameStrings.java + +commit e384bf02766b58fcd205471ed3907c6b64171765 +Author: Benjamin Trent +Date: Tue Jun 11 15:55:02 2019 -0500 + + [ML-DataFrame] stop task at completion of data frame function (#42955) (#43114) + + * stop data frame task after it finishes + + * test auto stop + + * adapt tests + + * persist the state correctly and move stop into listener + + * Calling `onStop` even if persistence fails, changing `stop` to rely on doSaveState + +commit 30d8085d969849a05454175fc6fb25e82acd8c89 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jun 11 19:46:03 2019 +0200 + + scheduleAtFixedRate would hang (#42993) + + Though not in use in elasticsearch currently, it seems surprising that + ThreadPool.scheduler().scheduleAtFixedRate would hang. A recurring + scheduled task is never completed (except on failure) and we test for + exceptions using RunnableFuture.get(), which hangs for periodic tasks. + Fixed by checking that task is done before calling .get(). + +commit 172cd4dbfa29478d4de3844611ac698351c4e756 +Author: Ryan Ernst +Date: Tue Jun 11 09:20:55 2019 -0700 + + Remove description from xpack feature sets (#43065) + + The description field of xpack featuresets is optionally part of the + xpack info api, when using the verbose flag. However, this information + is unnecessary, as it is better left for documentation (and the existing + descriptions describe anything meaningful). This commit removes the + description field from feature sets. + +commit 6108899a2d6d7a6436d3f6c1e66e60427d3803eb +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jun 11 17:40:40 2019 +0200 + + Fix unresponsive network simulation (#42579) + + Unresponsive network simulation would throw away requests. However, then + we no longer have any guarantees that a transport action either succeeds + or fails, which could lead to hangs (example: unclosed IndexShard + permits). + + Closes #42244 + +commit d3136f99e67b933eb878cf059fd26c890e1f17c7 +Author: David Roberts +Date: Tue Jun 11 16:38:35 2019 +0100 + + [ML] Fix race condition when closing time checker (#43098) + + The tests for the ML TimeoutChecker rely on threads + not being interrupted after the TimeoutChecker is + closed. This change ensures this by making the + close() and setTimeoutExceeded() methods synchronized + so that the code inside them cannot execute + simultaneously. + + Fixes #43097 + +commit b17fbe29337dc4cae4f05fd7f92fb3ed21d7bdc0 +Author: markharwood +Date: Tue Jun 11 16:23:04 2019 +0100 + + Docs enhancement for quote_field_suffix. (#43093) + + * Docs enhancement for quote_field_suffix. + Mentions the use of a fall-back field when specified field is missing. + Closes #40778 + +commit 0eeb080a72b64a56b498345b55845a4532fbaabe +Author: Jason Tedor +Date: Tue Jun 11 10:19:54 2019 -0400 + + Do not use runtime Java when starting BWC nodes (#43071) + + When starting BWC nodes, it could be that runtime Java home is set. Yet, + runtime Java home can advance beyond what a BWC node might be compatible + with. For example, if runtime Java home is set to JDK 13 and we are + starting a 7.1.2 node, we do not have any guarantees that 7.1.2 is + compatible with JDK 13 (since we never did any work to make it so). This + will continue to be the case as JDK releases advance, but we still need + to test against BWC nodes. This commit stops applying runtime Java home + when starting a BWC node. Instead, we would use the bundled JDK. + +commit 2c9a6cbf6980066a2a5bbe0c10c2d6029529c706 +Author: Martijn Laarman +Date: Tue Jun 11 15:15:11 2019 +0200 + + Documents the new deprecations options on the rest-api-spec (#41444) (#43090) + + * Documents the new deprecations options on the rest-api-spec + + Relates #41439 #38613 #35262 + + * remove reference to path now that #41452 is merged, also fixed missing a comma rendering the example json invalid + + * removed one more instance of path + + * make sure json examples are self contained and not excerpts + + (cherry picked from commit 4430f99750a3bf98373d69d2be59d71475c7aaad) + +commit 04cde1d6e22f1309034d33a701091fc0d108d0fa +Author: David Turner +Date: Tue Jun 11 12:16:01 2019 +0100 + + Defer reroute when nodes join (#42855) + + Today the master eagerly reroutes the cluster as part of processing node joins. + However, it is not necessary to do this reroute straight away, and it is + sometimes preferable to defer it until later. For instance, when the master + wins its election it processes joins and performs a reroute, but it would be + better to defer the reroute until after the master has become properly + established. + + This change defers this reroute into a separate task, and batches multiple such + tasks together. + +commit 5d3849215ba5a4dfd52c393aa5dbad7058fd0d62 +Author: Nhat Nguyen +Date: Tue Jun 11 06:39:43 2019 -0400 + + CCR should not replicate private/internal settings (#43067) + + With this change, CCR will not replicate internal or private settings to + follower indices. + + Closes #41268 + +commit cb7ce865b71dc5880a95b75e7782c63eb01bd570 +Author: Martijn Laarman +Date: Tue Jun 11 12:52:36 2019 +0200 + + remove path from rest-api-spec (#41452) (#43084) + + (cherry picked from commit f5fde1d0843d2f0f53d3b9a15b9cfc8b94471ab7) + +commit 1c7cd093752c363c122b2e8f4aabb4468125cae1 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jun 11 12:38:11 2019 +0200 + + Enable TRACE for testRecoverBrokenIndexMetadata (#43081) + + Relates to #43034 + +commit 900eb4f8826d115e770cefeb87cf5ea2135cdc67 +Author: Jim Ferenczi +Date: Tue Jun 11 11:53:04 2019 +0200 + + Handle empty terms index in TermsSliceQuery (#43078) + + #40741 introduced a merge policy that can drop the postings for the `_id` + field on soft deleted documents. The TermsSliceQuery assumes that every document + has has an entry in the postings for that field so it doesn't check if the terms + index exists or not. This change fixes this bug by checking if the terms index for + the `_id` field is null and ignore the segment entirely if it's the case. This should + be harmless since segments without an `_id` terms index should only contain soft deleted + documents. + + Closes #42996 + +commit 8de65daa450187597020986d7b4d03a2dd7ae85c +Author: Andrei Stefan +Date: Tue Jun 11 12:38:26 2019 +0300 + + Rename TESTRESPONSE[_cat] to TESTRESPONSE[non_json] (#43087) + + (cherry picked from commit 897b24e0563f59c03e85096fdb64cbc1dd1a5d60) + +commit 5b35ec1d9b53a399caadb25f0020c57f12538251 +Author: Andrei Stefan +Date: Tue Jun 11 12:04:00 2019 +0300 + + Restructure the SQL Language section to have proper sub-sections (#43007) + + Rest docs page update + - have the section be on separate pages + - add an Overview page + - add other formats examples + + (cherry picked from commit 309bd691ff3f8625f67ca09fc1dd8e265f7e6c92) + +commit 4a3287836d479f34558356363ff2b23f58332602 +Author: Andrei Stefan +Date: Tue Jun 11 12:02:46 2019 +0300 + + SQL: Clarify that the connections the jdbc driver creates are not pooled (#42992) + + (cherry picked from commit 406d5281bdfe682fb7ec9fefcdb61cce1b9e7270) + +commit 1776d6e0552bfa749ddfc7a78d46564f1583d462 +Author: Ioannis Kakavas +Date: Tue Jun 11 10:55:50 2019 +0300 + + Refresh remote JWKs on all errors (#42850) + + It turns out that key rotation on the OP, can manifest as both + a BadJWSException and a BadJOSEException in nimbus-jose-jwt. As + such we cannot depend on matching only BadJWSExceptions to + determine if we should poll the remote JWKs for an update. + + This has the side-effect that a remote JWKs source will be polled + exactly one additional time too for errors that have to do with + configuration, or for errors that might be caused by not synched + clocks, forged JWTs, etc. ( These will throw a BadJWTException + which extends BadJOSEException also ) + +commit 6a77dde5eab49322099ec9580f8321e6b88a89cd +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jun 11 09:48:07 2019 +0200 + + Better test diag output on OOM (#42989) + + If linearizability checking fails with OOM (or other exception), we did + not get the serialized history written into the log, making it difficult + to debug in cases where the problem is hard to reproduce. Fixed to + always attempt dumping the serialized history. + + Related to #42244 + +commit 79052050bfbb6b14ced66e5447724c19bfc0515b +Author: Benjamin Trent +Date: Mon Jun 10 21:52:53 2019 -0500 + + [ML] Adding support for geo_shape, geo_centroid, geo_point in datafeeds (#42969) (#43069) + + * [ML] Adding support for geo_shape, geo_centroid, geo_point in datafeeds + + * only supporting doc_values for geo_point fields + + * moving validation into GeoPointField ctor + +commit eadfe05587ebd4a3d35cd1dada1bb2f10078d228 +Author: Benjamin Trent +Date: Mon Jun 10 21:52:22 2019 -0500 + + [ML] Changes slice specification to auto. See #42996 (#43039) (#43070) + +commit 53eb6307009b9890221eda4ab08300ab69a84057 +Author: Nhat Nguyen +Date: Mon Jun 10 17:34:56 2019 -0400 + + Fix NPE in CcrRetentionLeaseIT (#43059) + + The retention leases stats is null if the processing shard copy is being + closed. In this the case, we should check against null then retry to + avoid failing a test. + + Closes #41237 + +commit f2e66e22eb0a149a9a96870ec8cd5426f9dc221e +Author: Nhat Nguyen +Date: Mon Jun 10 12:31:02 2019 -0400 + + Increase waiting time when check retention locks (#42994) + + WriteActionsTests#testBulk and WriteActionsTests#testIndex sometimes + fail with a pending retention lock. We might leak retention locks when + switching to async recovery. However, it's more likely that ongoing + recoveries prevent the retention lock from releasing. + + This change increases the waiting time when we check for no pending + retention lock and also ensures no ongoing recovery in + WriteActionsTests. + + Closes #41054 + +commit 4191df6e1dfcfe1496ffd53f9629eba0d8c4501e +Author: Nhat Nguyen +Date: Mon Jun 10 17:31:47 2019 -0400 + + Unmute IndexFollowingIT#testFollowIndex + + Fixed in #41987 + +commit 0a95b8c24d2327c8bb02808a26a784b0d3ce027f +Author: Benjamin Trent +Date: Mon Jun 10 16:47:40 2019 -0500 + + Fixing handling of auto slices in bulk scroll requests (#43050) (#43063) + + * Fixing handling of auto slices in bulk scroll requests + + * adjusting assertions for tests + +commit 1ddc4c8fc6fcd84aac92ddaa59d7cb13b3bcd775 +Author: Benjamin Trent +Date: Mon Jun 10 13:40:55 2019 -0500 + + [ML][Data Frame] Removes slice specification from DBQ. See #42996 (#43036) (#43055) + +commit f150443d9a542062ad6f72624a51273bb0fadc6e +Author: Jay Modi +Date: Mon Jun 10 11:44:34 2019 -0600 + + Default distro run creates elastic-admin user (#43004) + + When using gradle run by itself, this uses the default distro with a + basic license and enables security. There is a setup command to create + a elastic-admin user but only when the license is a trial license. Now + that security is available with the basic license, we should always run + this command when using the default distribution. + +commit 437d2d6d9f68ab87805a4f52b17ad42725310d6c +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jun 10 19:22:21 2019 +0200 + + Rename processor test fix (#43035) + + If the source field name is a prefix of the target field name, the + source field still exists after rename processor has run. Adjusted test + case to handle that case. + +commit 76a92b49a8b64277a2cf9ae5d053921e3835b8fd +Author: Dimitris Athanasiou +Date: Mon Jun 10 19:50:19 2019 +0300 + + [ML] Get resources action should be lenient when sort field is unmapped (#42991) (#43046) + + Get resources action sorts on the resource id. When there are no resources at + all, then it is possible the index does not contain a mapping for the resource + id field. In that case, the search api fails by default. + + This commit adjusts the search request to ignore unmapped fields. + + Closes elastic/kibana#37870 + +commit 014bad1f61244a6cce846f9342abd9b3b86aff82 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Jun 10 17:39:01 2019 +0200 + + Mute AzureDiscoveryClusterFormationTests (#43049) + + Relates #43048 + +commit bf5d56053a7a9f34cbfba612eef17e567bf14070 +Author: David Roberts +Date: Mon Jun 10 15:39:28 2019 +0100 + + [TEST] Adding a BWC test for ML categorization config (#42988) + + This test coverage was previously missing. + + Backport of #42981 + +commit 591372378891fa4641f34620d2cac77d00dd10bc +Author: James Rodewig +Date: Mon Jun 10 09:33:32 2019 -0400 + + [DOCS] Change `// TESTRESPONSE[_cat]` to `// TESTRESPONSE[non_json]` (#43006) + +commit 8e23e4518afa4131a50c09cb3692dcc4239ba472 +Author: Alan Woodward +Date: Mon Jun 10 10:26:15 2019 +0100 + + Move construction of custom analyzers into AnalysisRegistry (#42940) + + Both TransportAnalyzeAction and CategorizationAnalyzer have logic to build + custom analyzers for index-independent analysis. A lot of this code is duplicated, + and it requires the AnalysisRegistry to expose a number of internal provider + classes, as well as making some assumptions about when analysis components are + constructed. + + This commit moves the build logic directly into AnalysisRegistry, reducing the + registry's API surface considerably. + +commit 81a3b6e2fe61da641ed75a1aa0b1000b5970d7dd +Author: Mayya Sharipova +Date: Mon Jun 10 06:26:56 2019 -0400 + + Improve documentation for smart_cn analyzer (#42822) + +commit 9def454ea974d5bb2b61a96fa71a3bb581f9f727 +Author: Alpar Torok +Date: Mon Jun 10 13:39:39 2019 +0300 + + Clean up configuration when docker isn't available (#42745) + + We initially added `requireDocker` for a way for tasks to say that they + absolutely must have it, like the build docker image tasks. + Projects using the test fixtures plugin are not in this both, as the + intent with these is that they will be skipped if docker and docker-compose + is not available. + + Before this change we were lenient, the docker image build would succeed + but produce nothing. The implementation was also confusing as it was not + immediately obvious this was the case due to all the indirection in the + code. + + The reason we have this leniency is that when we added the docker image + build, docker was a fairly new requirement for us, and we didn't have + it deployed in CI widely enough nor had CI configured to prefer workers + with docker when possible. We are in a much better position now. + The other reason was other stack teams running `./gradlew assemble` + in their respective CI and the possibility of breaking them if docker is + not installed. We have been advocating for building specific distros for + some time now and I will also send out an additional notice + + The PR also removes the use of `requireDocker` from tests that actually + use test fixtures and are ok without it, and fixes a bug in test + fixtures that would cause incorrect configuration and allow some tasks + to run when docker was not available and they shouldn't have. + + Closes #42680 and #42829 see also #42719 + +commit 44aedcf97a441d8d92d33124047b570525881d1e +Author: Sachin Frayne <32795683+sachinfrayne@users.noreply.github.com> +Date: Mon Jun 10 11:36:10 2019 +0100 + + Correct the description of generate_word_parts (#43026) + +commit 12962ee0a7f58183a73b259c6cf0317fb10c6e41 +Author: Sam Mingo +Date: Mon Jun 10 05:12:22 2019 -0400 + + Update search-settings.asciidoc (#43016) + + Grammar and spelling fixes + +commit b2e7045b50150e3976621f0eddadbf0ebb9cbf13 +Author: Shubham Vipul Majmudar +Date: Mon Jun 10 02:11:09 2019 -0700 + + Update regexp-syntax.asciidoc (#43021) + + Corrects a typo. + +commit a5d4591af7d2a535193afd611fd5333f6e1bf15d +Author: Alpar Torok +Date: Mon Jun 10 11:08:34 2019 +0300 + + Don't run build-tools integ tests on FIPS (#42986) + + These run Gradle and FIPS isn't supported + + Closes #41721 + +commit 68339f90e9b2ec84c70f4f064f9f0599791b265f +Author: David Turner +Date: Sat Jun 8 23:10:33 2019 +0100 + + Mute AutodetectMemoryLimitIT#testTooManyPartitions + + Relates #43013 + +commit 39cb1abc9dac88b677aef792b4f52df7db48a94b +Author: Jim Ferenczi +Date: Mon Jun 10 09:31:23 2019 +0200 + + Fix auto fuzziness in query_string query (#42897) + + Setting `auto` after the fuzzy operator (e.g. `"query": "foo~auto"`) in the `query_string` + does not take the length of the term into account when computing the distance and always use + a max distance of 1. This change fixes this disrepancy by ensuring that the term is passed when + the fuzziness is computed. + +commit 25218733e6b0398ad88c758d0e334f6759f8a168 +Author: Vigya Sharma +Date: Mon Jun 10 12:56:13 2019 +0530 + + Allow routing commands with ?retry_failed=true (#42658) + + We respect allocation deciders, including the `MaxRetryAllocationDecider`, when + executing reroute commands. If you specify `?retry_failed=true` then the retry + counter is reset, but today this does not happen until after trying to execute + the reroute commands. This means that if an allocation has repeatedly failed, + but you want to take control and assign a shard to a particular node to work + around the repeated failures, you cannot execute the routing command in the + same call to `POST /_cluster/reroute` as the one that resets the failure + counter. + + This commit fixes this by resetting the failure counter first, meaning that you + can now explicitly allocate a repeatedly-failed shard like this: + + ``` + POST /_cluster/reroute?retry_failed=true + { + "commands": [ + { + "allocate_replica": { + "index": "blahblah", + "shard": 2, + "node": "node-4" + } + } + ] + } + ``` + + Fixes #39546 + +commit 036f9c4a557b8c32b1a5f609dc74065ff3a3ea1e +Author: Andrei Stefan +Date: Mon Jun 10 09:01:42 2019 +0300 + + SQL: cover the Integer type when extracting values from _source (#42859) + + * Take into consideration a wider range of Numbers when extracting the + values from source, more specifically - BigInteger and BigDecimal. + + (cherry picked from commit 561b8d73dd7b03c50242e4e3f0128b2142959176) + +commit 90485c60281b3684e5cb4d07032bd529eaf14abc +Author: Andrei Stefan +Date: Mon Jun 10 09:00:39 2019 +0300 + + Since SQL is GA, remove the sql language plugin from this list (#41533) + + (cherry picked from commit f715d722e8df54b3d3fe84d3ff57dfd6a198a2ac) + +commit b96ed1f9f76dfbe9bed39f177c2aab137259c463 +Author: Jason Tedor +Date: Sun Jun 9 22:56:24 2019 -0400 + + Add note to CCR docs about mapping/alias updates + + This commit adds a note to the docs clarifying that it is not possible + to manually update the mapping nor the aliases of a follower index. + +commit 25ca315d78973a4db67dfc8c21e39ac1e8e35d68 +Author: Jason Tedor +Date: Sun Jun 9 22:54:27 2019 -0400 + + Add note to CCR docs regarding alias replication + + This commit adds a note to the docs regarding the automatic replication + of aliases by a follower index from its leader index. + +commit 63bad28005fb445152247d18cb2369f72cc30808 +Author: Jason Tedor +Date: Sun Jun 9 22:43:55 2019 -0400 + + Do not allow modify aliases on followers (#43017) + + Now that aliases are replicated by a follower from its leader, this + commit prevents directly modifying aliases on follower indices. + +commit 0ebcb21d2c76e54bade65976face5c2c28acc5f0 +Author: Nhat Nguyen +Date: Sat Jun 8 11:08:27 2019 -0400 + + Unmuted testRecoverBrokenIndexMetadata + + These tests should be okay as we flush at the end of peer recovery. + + Closes #40867 + +commit afe65b5988e78379ab6ec40b285eb0ccb2b3aeda +Author: Nhat Nguyen +Date: Sat Jun 8 21:50:37 2019 -0400 + + Fix assertion in ReadOnlyEngine (#43010) + + We should execute the assertion before throwing an exception; + otherwise, it's a noop. + +commit 915d2f2daadb35c8507c7e79bd658928ef26fc48 +Author: Jason Tedor +Date: Sun Jun 9 10:06:14 2019 -0400 + + Refactor put mapping request validation for reuse (#43005) + + This commit refactors put mapping request validation for reuse. The + concrete case that we are after here is the ability to apply effectively + the same framework to indices aliases requests. This commit refactors + the put mapping request validation framework to allow for that. + +commit 0a982fc57fa57279138865c361e4a712148d64a6 +Author: Nhat Nguyen +Date: Sat Jun 8 00:29:39 2019 -0400 + + Mute testLookupSeqNoByIdInLucene + + Tracked at #42979 + +commit 5f7d5920d7d8e11863819a675c6f86ea995c92b2 +Author: Jason Tedor +Date: Fri Jun 7 22:59:53 2019 -0400 + + Fix IOUtils#fsync on Windows fsyncing directories (#43008) + + Fsyncing directories on Windows is not possible. We always suppressed + this by allowing that an AccessDeniedException is thrown when attemping + to open the directory for reading. Yet, this suppression also allowed + other IOExceptions to be suppressed, and that was a bug (e.g., the + directory not existing, or a filesystem error and reasons that we might + get an access denied there, like genuine permissions issues). This + leniency was previously removed yet it exposed that we were suppressing + this case on Windows. Rather than relying on exceptions for flow control + and continuing to suppress there, we simply return early if attempting + to fsync a directory on Windows (we will not put this burden on the + caller). + +commit b580677412a81386ac3bf286752fb2f49aea5003 +Author: Jason Tedor +Date: Fri Jun 7 17:46:12 2019 -0400 + + Fix put mapping request validators random test + + This commit fixes a test bug in the request validators random test. In + particular, an assertion was not properly nested in a guard that would + ensure that was at least one failure. + + Relates #43000 + +commit 553c73b22d165f8b10a5ee0bf59cab5d21650315 +Author: Benjamin Trent +Date: Fri Jun 7 15:43:06 2019 -0500 + + [ML][Data Frame] allow null values for aggs with sparse data (#42966) (#42998) + + * [ML][Data Frame] allow null values for aggs with sparse data + + * Making classes static, memory allocation optimization + +commit d6fe4b648d00463c9bcab28a1b9ca54a03ea1309 +Author: Jason Tedor +Date: Fri Jun 7 16:10:10 2019 -0400 + + Fix possible NPE in put mapping validators (#43000) + + When applying put mapping validators, we apply all the validators in the + collection. If a failure occurs, we collect that as a top-level + exception, and suppress any additional failures into the top-level + exception. However, if a request passes the validator after a top-level + exception has been collected, we would try to suppress a null exception + into the top-level exception. This is a violation of the + Throwable#addSuppressed API. This commit addresses this, and adds test + to cover the logic of collecting the failures when validating a put + mapping request. + +commit 5bc0dfce942a0d04176c1b2390792d842c310894 +Author: David Turner +Date: Fri Jun 7 20:28:02 2019 +0100 + + Improve translog corruption detection (#42980) + + Today we test for translog corruption by incrementing a byte by 1 somewhere in + a file, and verify that this leads to a `TranslogCorruptionException`. + However, we rely on _all_ corruptions leading to this exception in the + `RemoveCorruptedShardDataCommand`: this command fails if a translog file + corruption leads to a different kind of exception, and `EOFException` and + `NegativeArraySizeException` are both possible. This commit strengthens the + translog corruption detection tests by simulating the following: + + - a random value is written + - the file is truncated + + It also makes sure that we return a `TranslogCorruptionException` in all such + cases. + + Fixes #42661 + Backport of #42744 + +commit 479a1eeff637c8c10cdc05da2847c80ae3bdf78b +Author: Jason Tedor +Date: Fri Jun 7 15:21:41 2019 -0400 + + Drop dead code for socket permissions for transport (#42990) + + This code has not been needed since the removal of tribe nodes, it was + left behind when those were dropped (note that regular transport + permissions are handled through transport profiles, even if they are not + explicitly in use). + +commit 5342616a23abd0e65c0aa9e4d8967fc5e389fbc5 +Author: James Rodewig +Date: Fri Jun 7 11:22:32 2019 -0400 + + [DOCS] Add explicit `articles_case` parameter to Elision Token Filter example (#42987) + +commit 755ba72896536972bb32c3d7daf55de5fdf4c51a +Author: Benjamin Trent +Date: Fri Jun 7 10:13:18 2019 -0500 + + [ML][Data frame] make sure that fields exist when creating progress (#42943) (#42984) + +commit 0719779a481bfe37d0301dec858bef5f25f6598b +Author: markharwood +Date: Fri Jun 7 14:53:17 2019 +0100 + + Search - enable low_level_cancellation by default. (#42291) (#42857) + + Benchmarking on worst-case queries (max agg on match_all or popular-term query with large index) was not noticeably slower. + + Closes #26258 + +commit b77be89c9a7483ec993b0ad98521814e019b3c95 +Author: jalvar08 +Date: Fri Jun 7 08:17:36 2019 -0400 + + Remove Comma in Example (#41873) + + The comma is there in error as there are no other parameter after 'value' + +commit 45bbd7f7f1ea1f5bd547f3b89b6351167057965b +Author: Jason Tedor +Date: Fri Jun 7 08:34:19 2019 -0400 + + Only ignore IOException when fsyncing on dirs (#42972) + + Today in the method IOUtils#fsync we ignore IOExceptions when fsyncing a + directory. However, the catch block here is too broad, for example it + would be ignoring IOExceptions when we try to open a non-existant + file. This commit addresses that by scoping the ignored exceptions only + to the invocation of FileChannel#force. + +commit dea935ac3165cfb0106e386de0efef14b3beb03a +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jun 7 12:16:36 2019 +0200 + + Reindex max_docs parameter name (#42942) + + Previously, a reindex request had two different size specifications in the body: + * Outer level, determining the maximum documents to process + * Inside the source element, determining the scroll/batch size. + + The outer level size has now been renamed to max_docs to + avoid confusion and clarify its semantics, with backwards compatibility and + deprecation warnings for using size. + Similarly, the size parameter has been renamed to max_docs for + update/delete-by-query to keep the 3 interfaces consistent. + + Finally, all 3 endpoints now support max_docs in both body and URL. + + Relates #24344 + +commit 5929803413b8bc3164ed517aace93c0d0ea1b4de +Author: David Turner +Date: Fri Jun 7 10:38:19 2019 +0100 + + Relax timeout in NodeConnectionsServiceTests (#42934) + + Today we assert that the connection thread is blocked by the time the test gets + to the barrier, but in fact this is not a valid assertion. The following + `Thread.sleep()` will cause the test to fail reasonably often. + + ```diff + diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java + index 193cde3180d..0e57211cec4 100644 + --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java + +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java + @@ -364,6 +364,7 @@ public class NodeConnectionsServiceTests extends ESTestCase { + final CheckedRunnable connectionBlock = nodeConnectionBlocks.get(node); + if (connectionBlock != null) { + try { + + Thread.sleep(50); + connectionBlock.run(); + } catch (Exception e) { + throw new AssertionError(e); + ``` + + This change relaxes the test to allow some time for the connection thread to + hit the barrier. + + Fixes #40170 + +commit 090d42d3e6c2181d12a92e856ee818dbdef0259a +Author: Tim Vernum +Date: Fri Jun 7 14:18:05 2019 +1000 + + Permit API Keys on Basic License (#42973) + + Kibana alerting is going to be built using API Keys, and should be + permitted on a basic license. + + This commit moves API Keys (but not Tokens) to the Basic license + + Relates: elastic/kibana#36836 + Backport of: #42787 + +commit 667c613d9efa5929aa8de3fb84069feb5b05b650 +Author: Tim Brooks +Date: Thu Jun 6 17:44:40 2019 -0400 + + Remove `nonApplicationWrite` from `SSLDriver` (#42954) + + Currently, when the SSLEngine needs to produce handshake or close data, + we must manually call the nonApplicationWrite method. However, this data + is only required when something triggers the need (starting handshake, + reading from the wire, initiating close, etc). As we have a dedicated + outbound buffer, this data can be produced automatically. Additionally, + with this refactoring, we combine handshake and application mode into a + single mode. This is necessary as there are non-application messages that + are sent post handshake in TLS 1.3. Finally, this commit modifies the + SSLDriver tests to test against TLS 1.3. + +commit 61b62125b85ce8f542154bf005ac6001905f81e6 +Author: henryptung +Date: Thu Jun 6 11:21:16 2019 -0700 + + Wire query cache into sorting nested-filter computation (#42906) + + Don't use Lucene's default query cache when filtering in sort. + + Closes #42813 + +commit d18f511327752562eaa6fcf9c0a91c15a1b8ae64 +Author: Tim Brooks +Date: Thu Jun 6 14:50:06 2019 -0400 + + Propogate version in reindex from remote search (#42958) + + This is related to #31908. In order to use the external version in a + reindex from remote request, the search request must be configured to + request the version (as it is not returned by default). This commit + modifies the search request to request the version. Additionally, it + modifies our current reindex from remote tests to randomly use the + external version_type. + +commit ca5dbf93a5c062ce9a039c84890f506c7dc731ff +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jun 6 19:48:59 2019 +0200 + + Fix concurrent search and index delete (#42621) + + Changed order of listener invocation so that we notify before + registering search context and notify after unregistering same. + + This ensures that count up/down like what we do in ShardSearchStats + works. Otherwise, we risk notifying onFreeScrollContext before notifying + onNewScrollContext (same for onFreeContext/onNewContext, but we + currently have no assertions failing in those). + + Closes #28053 + +commit 2de919e3a81944ff1eee9633b603bae7a86af4bc +Author: James Rodewig +Date: Thu Jun 6 10:45:04 2019 -0400 + + [DOCS] Move 'Scripting' section to top-level navigation. (#42939) + +commit e35b240a43c1dae5541abbdff61cdcc966aeee64 +Author: Gordon Brown +Date: Thu Jun 6 08:28:32 2019 -0600 + + Fix hang in test for "too many fields" dep. check (#42909) + + This commit fixes a rare case where the method to randomly generate a + very large mapping could enter an infinite loop. + +commit 1f1868a294b89f6d976c37269823b510477bafbb +Author: Benjamin Trent +Date: Thu Jun 6 08:51:11 2019 -0500 + + [ML][Data Frame] pull state and states for indexer from index (#42856) (#42935) + + * [ML][Data Frame] pull state and states for indexer from index + + * Update DataFrameTransformTask.java + +commit 02e6acf2d21a1b0c05c151fd315edb61cdf1c380 +Author: Benjamin Trent +Date: Thu Jun 6 08:33:49 2019 -0500 + + [ML] [Data Frame] Adding pending task wait to the hlrc cleanup (#42907) (#42930) + +commit ed186b4485ae1af680ad539acd0ce722be9df40a +Author: James Rodewig +Date: Thu Jun 6 08:32:42 2019 -0400 + + [DOCS] Rewrite terms query (#42889) + +commit 7fcca55a3c6bc5ad630dfb3b32350fc722f16e0e +Author: Simon Willnauer +Date: Thu Jun 6 14:17:44 2019 +0200 + + [TEST] Remove unnecessary log line + +commit 2582e1e8adcb8de02442bf84e058e05631622053 +Author: Simon Willnauer +Date: Thu Jun 6 14:14:41 2019 +0200 + + Fix `InternalEngineTests#testPruneAwayDeletedButRetainedIds` + + The test failed because we had only a single document in the index + that got deleted such that some assertions that expected at least + one live doc failed. + + Relates to: #40741 + +commit 9f7be70f7af27e64e5f94479712ccefb98c96bc0 +Author: Yannick Welsch +Date: Thu Jun 6 14:05:17 2019 +0200 + + Fix testPendingTasks (#42922) + + Fixes a race in the test which can be reliably reproduced by adding Thread.sleep(100) to the end of + IndicesService.processPendingDeletes + + Closes #18747 + +commit 72735be673b5ca0bff72944ccc2a61551cdf6af8 +Author: Yannick Welsch +Date: Thu Jun 6 14:04:45 2019 +0200 + + Fix NPE when rejecting bulk updates (#42923) + + Single updates use a different internal code path than updates that are wrapped in a bulk request. + While working on a refactoring to bring both closer together I've noticed that bulk updates were + failing some of the tests that single updates passed. In particular, bulk updates cause + NullPointerExceptions to be thrown and listeners not being properly notified when being rejected + from the thread pool. + +commit ba8bd8dfbe089c2cb3444efeb38d69665d2a7766 +Author: Hendrik Muhs +Date: Thu Jun 6 14:03:49 2019 +0200 + + configure auto expand for dataframe indexes (#42924) + + creates the dataframe destination index with auto expand for replicas (0-1) + +commit 2c3bd32aff353bffc586b3ceae09b1fc2c533694 +Author: Simon Willnauer +Date: Thu Jun 6 13:18:48 2019 +0200 + + Add a merge policy that prunes ID postings for soft-deleted but retained documents (#40741) + + This change adds a merge policy that drops all _id postings for documents that + are marked as soft-deleted but retained across merges. This is usually unnecessary + unless soft-deletes are used with a retention policy since otherwise a merge would + remove deleted documents anyway. + + Yet, this merge policy prevents extreme cases where a very large number of soft-deleted + documents are retained and are impacting update performance. + Note, using this merge policy will remove all lookup by ID capabilities for soft-deleted documents. + +commit ba5b1d3249295928dd7d28877969e94e7c6f98e6 +Author: Ioannis Kakavas +Date: Thu Jun 6 14:27:48 2019 +0300 + + Mute failing testPerformActionAttrsRequestFails (#42933) + +commit 6ee578c6eb3f2a6164934720ee82e2456d7bb226 +Author: Ioannis Kakavas +Date: Thu Jun 6 14:00:00 2019 +0300 + + Mute testEnableDisableBehaviour (#42929) + +commit 40c827a3b80e0664a22f97184e24efdd2a33feaa +Author: David Roberts +Date: Thu Jun 6 11:00:34 2019 +0100 + + [ML] Close sample stream in find_file_structure endpoint (#42896) + + A static code analysis revealed that we are not closing + the input stream in the find_file_structure endpoint. + This actually makes no difference in practice, as the + particular InputStream implementation in this case is + org.elasticsearch.common.bytes.BytesReferenceStreamInput + and its close() method is a no-op. However, it is good + practice to close the stream anyway. + +commit b202a59f88e45cde452136b90e9d5ba25e63ca3a +Author: David Roberts +Date: Thu Jun 6 08:56:57 2019 +0100 + + [ML] Add earliest and latest timestamps to field stats (#42890) + + This change adds the earliest and latest timestamps into + the field stats for fields of type "date" in the output of + the ML find_file_structure endpoint. This will enable the + cards for date fields in the file data visualizer in the UI + to be made to look more similar to the cards for date + fields in the index data visualizer in the UI. + +commit 280a2c940127fc50885defce2aaf2403b753774c +Author: Hendrik Muhs +Date: Thu Jun 6 07:45:36 2019 +0200 + + [ML-DataFrame] reduce log spam: do not trigger indexer if state is indexing or stopping (#42849) + + reduce log spam: do not trigger indexer if state is indexing or stopping + +commit 5cd503a1e022bdc7cbf2ab217a314eabc04b7fdd +Author: Hendrik Muhs +Date: Thu Jun 6 07:43:36 2019 +0200 + + [ML-DataFrame] increase the scheduler interval to 10s (#42845) + + increases the scheduler interval to fire less frequently, namely changing it from 1s to 10s. The scheduler interval is used for retrying after an error condition. + +commit 6eb4600e93ac1ee3ac027c3cdad9823b43048bf8 +Author: Gordon Brown +Date: Wed Jun 5 17:30:31 2019 -0600 + + Add custom metadata to snapshots (#41281) + + Adds a metadata field to snapshots which can be used to store arbitrary + key-value information. This may be useful for attaching a description of + why a snapshot was taken, tagging snapshots to make categorization + easier, or identifying the source of automatically-created snapshots. + +commit 1f4ff97d7dd7b272398b20ba90132685bc413c3c +Author: Mark Vieira +Date: Wed Jun 5 13:45:36 2019 -0700 + + Mute failing test + + (cherry picked from commit 4952d4facf5949abdb9aae47dbe1ee18cf7eef99) + +commit 12433434ef779421497908617eed6d2e8417fa61 +Author: Mark Vieira +Date: Wed Jun 5 13:28:43 2019 -0700 + + Skip installation of pre-bundled integ-test modules (#42900) + + (cherry picked from commit 67bedf6c7c5d6db1394c045958668ac930a92d57) + +commit 99542e66a647a5a62362eb28b631d223f7c5f179 +Author: Christoph Büscher +Date: Wed Jun 5 22:02:17 2019 +0200 + + [Docs] Clarify caveats for phonetic filters replace option (#42807) + + The `replace` option in the phonetic token filter can have suprising side + effects, e.g. such as described in #26921. This PR adds a note to be mindful + about such scenarios and offers alternatives to using the `replace` option. + + Closes #26921 + +commit 1300183001f27f53c9b74b3f6a3227a586516c01 +Author: James Baiera +Date: Wed Jun 5 16:03:07 2019 -0400 + + NullPointerException when creating a watch with Jira action (#41922) (#42081) (#42873) + + NullPointerException when secured_url does not use proper scheme in jira action. + This commit will handle Expection and display proper message. + +commit 757c6a45a0494a62f44bf26fcd76be69b2692830 +Author: Lisa Cawley +Date: Wed Jun 5 12:29:40 2019 -0700 + + [DOCS] Adds discovery.type (#42823) + + Co-Authored-By: David Turner + +commit ab5bc8359757aacdfcd16079c0847da8f82d5a6f +Author: Przemyslaw Gomulka +Date: Wed Jun 5 19:50:04 2019 +0200 + + Deprecation info for joda-java migration on 7.x (#42659) + + Some clusters might have been already migrated to version 7 without being warned about the joda-java migration changes. + Deprecation api on that version will give them guidance on what patterns need to be changed. + relates. This change is using the same logic like in 6.8 that is: verifying the pattern is from the incompatible set ('y'-Y', 'C', 'Z' etc), not from predifined set, not prefixed with 8. AND was also created in 6.x. Mappings created in 7.x are considered migrated and should not generate warnings + + There is no pipeline check (present on 6.8) as it is impossible to verify when the pipeline was created, and therefore to make sure the format is depracated or not + #42010 + +commit 790d2124f62984a6f31e89b4d1a6ca9899a57b83 +Author: Jack Conradson +Date: Wed Jun 5 10:20:46 2019 -0700 + + Clean Up Painless Datetime Docs (#42869) + + This change abstracts the specific types away from the different + representations of datetime as a datetime representation in code can be all + kinds of different things. This defines the three most common types of + datetimes as numeric, string, and complex while outlining the type most + typically used for these as long, String, and ZonedDateTime, respectively. + Documentation uses the definitions while examples use the types. This makes + the documentation easier to consume especially for people from a non-Java + background. + +commit 84eab4eba12308e90e70b215ab789c2bd3ce6377 +Author: Mark Vieira +Date: Wed Jun 5 10:08:30 2019 -0700 + + Omit JDK sources archive from bundled JDK (#42821) + + (cherry picked from commit 71d1454fe5ecc222801731a5f0e0e1053dc8997e) + +commit cfdb1b771ebf9b006e5a9eb4f7e52e8d47b5d70a +Author: Przemyslaw Gomulka +Date: Wed Jun 5 17:15:37 2019 +0200 + + Enable console audit logs for docker backport#42671 #42887 + + Enable audit logs in docker by creating console appenders for audit loggers. + also rename field @timestamp to timestamp and add field type with value audit + + The docker build contains now two log4j configuration for oss or default versions. The build now allows override the default configuration. + + Also changed the format of a timestamp from ISO8601 to include time zone as per this discussion #36833 (comment) + + closes #42666 + backport#42671 + +commit 0a9901925978d5a1c5bd8ebe01b7222e16920454 +Author: Jason Tedor +Date: Wed Jun 5 09:47:06 2019 -0400 + + Disable building on JDK 13 in CI + + This commit disables building on JDK 13 in CI. The reason for this is + because Gradle is not yet ready to run on JDK 13. We could re-introduce + infrastructure to enable Gralde to run on a different JDK than the build + JDK, but rather than introducing such complexity we will instead wait + for Gradle to be ready to run on JDK 13. + +commit 293f306b9add8e9afe7447f4e298f018afa2294f +Author: Benjamin Trent +Date: Wed Jun 5 07:09:34 2019 -0500 + + [ML][Data Frame] forcing that no ptask => STOPPED state (#42800) (#42860) + + * [ML][Data Frame] forcing that no ptask => STOPPED state + + * Addressing side-effect, early exit for stop when stopped + +commit d5baedb78963909c3f720373424dadc21267488a +Author: David Roberts +Date: Wed Jun 5 10:52:25 2019 +0100 + + [ML] Change dots in CSV column names to underscores (#42839) + + Dots in the column names cause an error in the ingest + pipeline, as dots are special characters in ingest pipeline. + This PR changes dots into underscores in CSV field names + suggested by the ML find_file_structure endpoint _unless_ + the field names are specifically overridden. The reason for + allowing them in overrides is that fields that are not + mentioned in the ingest pipeline can contain dots. But it's + more consistent that the default behaviour is to replace + them all. + + Fixes elastic/kibana#26800 + +commit ebec118ccf66e4d122558a1ff0231a9627e08f6a +Author: Simon Willnauer +Date: Wed Jun 5 12:10:02 2019 +0200 + + Bring back ExecutionException after backport + +commit d3524fdd0633d4b1f979429ca768d46b64a642de +Author: Simon Willnauer +Date: Wed Jun 5 11:25:19 2019 +0200 + + Add back import after backport + +commit 4dfaeb904660a3b7f55c03272414b8d78c4ce8c3 +Author: Simon Willnauer +Date: Wed Jun 5 11:24:58 2019 +0200 + + Remove post Java 9 API usage after backport + +commit de0ea4bbf77235fd25e9a340b648be981b9218f7 +Author: Jim Ferenczi +Date: Wed Jun 5 08:47:33 2019 +0200 + + Deduplicate alias and concrete fields in query field expansion (#42328) + + The full-text query parsers accept field pattern that are expanded using the mapping. + Alias field are also detected during the expansion but they are not deduplicated with the + concrete fields that are found from other patterns (or the same). This change ensures + that we deduplicate the target fields of the full-text query parsers in order to avoid + adding the same clause multiple times. Boolean queries are already able to deduplicate + clauses during rewrite but since we also use DisjunctionMaxQuery it is preferable to detect + these duplicates early on. + +commit 41a9f3ae3b3b45b0918fd79e91f18d66aa73665f +Author: Simon Willnauer +Date: Wed Jun 5 11:00:26 2019 +0200 + + Use reader attributes to control term dict memory useage (#42838) + + This change makes use of the reader attributes added in LUCENE-8671 + to ensure that `_id` fields are always on-heap for best update performance + and term dicts are generally off-heap on Read-Only engines. + + Closes #38390 + +commit 955aee8a07b11315ae88015998842f6e7f147174 +Author: David Turner +Date: Wed Jun 5 09:00:22 2019 +0100 + + More logging in testRerouteOccursOnDiskPassingHighWatermark (#42864) + + This test is failing because recoveries of these empty shards are not + completing in a reasonable time, but the reason for this is still obscure. This + commit adds yet more logging. + + Relates #40174, #42424 + +commit 00f01aaece6516c522ded37244ad9dd09cc56311 +Author: Dimitrios Liappis +Date: Mon Jun 3 15:12:29 2019 +0300 + + Clarify heap setting in Docker docs (#42754) + + Add note in the Docker docs that even when container memory is limited, + we still require specifying -Xms/-Xmx using one of the supported + methods. + +commit be2365811455efb6bf2628574380721aec7975d8 +Author: Mark Vieira +Date: Tue Jun 4 17:53:24 2019 -0700 + + Avoid clobbering shared testcluster JAR files when installing modules (#42879) + + (cherry picked from commit 6da9aa29170c840bba08637c106c85bf16359979) + +commit 78be3dde255b5899dd48c08cb74927b69486bc2c +Author: Jason Tedor +Date: Tue Jun 4 20:51:58 2019 -0400 + + Enable testing against JDK 13 EA builds (#40829) + + This commit adds JDK 13 to the CI rotation for testing. For now, we will + be testing against JDK 13 EA builds. + +commit 117df87b2b7304ee23a3f86e102f656a6df175fb +Author: Jason Tedor +Date: Tue Jun 4 20:36:24 2019 -0400 + + Replicate aliases in cross-cluster replication (#42875) + + This commit adds functionality so that aliases that are manipulated on + leader indices are replicated by the shard follow tasks to the follower + indices. Note that we ignore write indices. This is due to the fact that + follower indices do not receive direct writes so the concept is not + useful. + + Relates #41815 + +commit aad1b3a2a0d825c6ecb7e753772ca8ef55600760 +Author: Jason Tedor +Date: Tue Jun 4 18:21:20 2019 -0400 + + Fix version parsing in various tests (#42871) + + This commit fixes the version parsing in various tests. The issue here is that + the parsing was relying on java.version. However, java.version can contain + additional characters such as -ea for early access builds. See JEP 233: + + Name Syntax + ------------------------------ -------------- + java.version $VNUM(\-$PRE)? + java.runtime.version $VSTR + java.vm.version $VSTR + java.specification.version $VNUM + java.vm.specification.version $VNUM + + Instead, we want java.specification.version. + +commit 72eb9c2d44ba88a1592918491bd39c41feb84883 +Author: Albert Zaharovits +Date: Tue Jun 4 11:53:26 2019 -0700 + + Eclipse libs projects setup fix (#42852) + + Fallout from #42773 for eclipse users. + + (cherry picked from commit 998419c49fe51eb8343664a80f07d8d8d39abc6a) + +commit e44b8b1e2ecc1aa66cee037197ac369a2bb94518 +Author: Mark Vieira +Date: Tue Jun 4 13:50:23 2019 -0700 + + [Backport] Remove dependency substitutions 7.x (#42866) + + * Remove unnecessary usage of Gradle dependency substitution rules (#42773) + + (cherry picked from commit 12d583dbf6f7d44f00aa365e34fc7e937c3c61f7) + +commit 1514d1a1beec433aaf196bf00b158087023a959a +Author: Chris Cho +Date: Wed Jun 5 01:28:58 2019 +0900 + + Change shard allocation filter property and api (#42602) + + The current example is not working and a bit confused. This change tries + to match it with the sample of the watcher blog. + +commit 6391f90616cca5338f5a77b076df05297562de4d +Author: Andrey Ershov +Date: Tue Jun 4 17:22:33 2019 +0200 + + Fix testNoMasterActionsWriteMasterBlock (#42798) + + This commit performs the proper restore of network disruption. + Previously disruptionScheme.stopDisrupting() was called that does not + ensure that connectivity between cluster nodes is restored. The test + was checking that the cluster has green status, but it was not checking + that connectivity between nodes is restored. + Here we switch to internalCluster().clearDisruptionScheme(true) which + performs both checks before returning. + + Closes #39688 + + (cherry picked from commit c8988d5cf5a85f9b28ce148dbf100aaa6682a757) + +commit df124f32db8aba051e1b3d04e7150e10a1e3616c +Author: Alan Woodward +Date: Tue Jun 4 14:32:43 2019 +0100 + + Refactor control flow in TransportAnalyzeAction (#42801) + + The control flow in TransportAnalyzeAction is currently spread across two large + methods, and is quite difficult to follow. This commit tidies things up a bit, to make + it clearer when we use pre-defined analyzers and when we use custom built ones. + +commit 783159dcbc5a1dae5960b7f25ea1f05455b354a2 +Author: James Rodewig +Date: Tue Jun 4 09:39:08 2019 -0400 + + [DOCS] Fix typo in bucket script aggregation link + +commit d050c52fd1b6b055fb588ef2531bc689523426f2 +Author: James Rodewig +Date: Tue Jun 4 08:42:43 2019 -0400 + + [DOCS] Fix broken bucket script agg link + +commit d9c582e66bdecb9acc74a06cd49645dcabdc37cf +Author: Christoph Büscher +Date: Tue Jun 4 14:37:55 2019 +0200 + + [Docs] Add to preference parameter docs (#42797) + + Adding notes to the existing docs about how using `preference` might increase + request cache utilization but also add warning about the downsides. + + Closes #24278 + +commit 32eae0dfe93179502884fb9d07811e67ce5038a1 +Author: Benjamin Trent +Date: Tue Jun 4 07:19:58 2019 -0500 + + [ML] [Data Frame] Adding supported aggs in docs (#42728) (#42842) + + * [ML] [Data Frame] Adding supported aggs in docs + + * [DOCS] Moves pivot to definitions list + +commit 440ec4d9f53d42155a96b1ed16ee21c376c87d25 +Author: Ioannis Kakavas +Date: Tue Jun 4 14:08:41 2019 +0300 + + [Backport 7.x] OpenID Connect realm guide (#42836) + + This commit adds a configuration guide for the newly introduced + OpenID Connect realm. The guide is similar to the style of the + SAML Guide and shares certain parts where applicable (role mapping) + It also contains a short section on how the realm can be used for + authenticating users without Kibana. + + Co-Authored-By: Lisa Cawley + + Backport of #41423 and #42555 + +commit 928f49992f38660f1331c8d46bf5e415e4cccc54 +Author: Tim Vernum +Date: Tue Jun 4 19:48:37 2019 +1000 + + Don't require TLS for single node clusters (#42830) + + This commit removes the TLS cluster join validator. + + This validator existed to prevent v6.x nodes (which mandated + TLS) from joining an existing cluster of v5.x nodes (which did + not mandate TLS) unless the 6.x node (and by implication the + 5.x nodes) was configured to use TLS. + + Since 7.x nodes cannot talk to 5.x nodes, this validator is no longer + needed. + + Removing the validator solves a problem where single node clusters + that were bound to local interfaces were incorrectly requiring TLS + when they recovered cluster state and joined their own cluster. + + Backport of: #42826 + +commit 9f470c20ed81e1ba6010cf5f3ac723a6aa9e0b9e +Author: David Turner +Date: Mon Jun 3 17:20:47 2019 +0100 + + More improvements to cluster coordination docs (#42799) + + This commit addresses a few more frequently-asked questions: + + * clarifies that bootstrapping doesn't happen even after a full cluster + restart. + + * removes the example that uses IP addresses, to try and further encourage the + use of node names for bootstrapping. + + * clarifies that auto-bootstrapping might form different clusters on different + hosts, and gives a process for starting again if this wasn't what you wanted. + + * adds the "do not stop half-or-more of the master-eligible nodes" slogan that + was notably absent. + + * reformats one of the console examples to a narrower width + +commit 8de3a88205b33e9276555dfb6eed15bceee401fe +Author: Tim Vernum +Date: Tue Jun 4 14:25:43 2019 +1000 + + Log the status of security on license change (#42741) + + Whether security is enabled/disabled is dependent on the combination + of the node settings and the cluster license. + + This commit adds a license state listener that logs when the license + change causes security to switch state (or to be initialised). + + This is primarily useful for diagnosing cluster formation issues. + + Backport of: #42488 + +commit 9035e61825fdf446c7a2d89d08cdcae498e75aa3 +Author: Tim Vernum +Date: Tue Jun 4 14:25:20 2019 +1000 + + Detect when security index is closed (#42740) + + If the security index is closed, it should be treated as unavailable + for security purposes. + + Prior to 8.0 (or in a mixed cluster) a closed security index has + no routing data, which would cause a NPE in the cluster change + handler, and the index state would not be updated correctly. + This commit fixes that problem + + Backport of: #42191 + +commit 87cc6a974c13c65c3136432aa4c6e56ba958ac34 +Author: Benjamin Trent +Date: Mon Jun 3 19:49:58 2019 -0500 + + [ML] [Data Frame] adding and modifying auditor messages (#42722) (#42818) + + * [ML] [Data Frame] adding and modifying auditor messages + + * Update DataFrameTransformTask.java + +commit eab88354f20f9a93f02ee350ab1dcec0c09fc4e1 +Author: Marios Trivyzas +Date: Mon Jun 3 22:56:39 2019 +0200 + + [Docs] Fix reference to `boost` and `slop` params (#42803) + + For `multi_match` query: link `boost` param to the generic reference + for query usage and `slop` to the `match_phrase` query where its usage + is documented. + + Fixes: #40091 + (cherry picked from commit 69993049a8bd9e7f042935729fe69a8266d95a0a) + +commit de72fe344c21f97774ff8861213cf74a1f2e7bf3 +Author: Jack Conradson +Date: Mon Jun 3 13:22:45 2019 -0700 + + Add Basic Date Docs to Painless (#42544) + +commit 3b42dde64fa7165163d98ebbf54a9978d0175c23 +Author: Marios Trivyzas +Date: Mon Jun 3 22:26:01 2019 +0200 + + [Docs] Add note for date patterns used for index search. (#42810) + + Add an explanatory NOTE section to draw attention to the difference + between small and capital letters used for the index date patterns. + e.g.: HH vs hh, MM vs mm. + + Closes: #22322 + (cherry picked from commit c8125417dc33215651f9bb76c9b1ffaf25f41caf) + +commit 415f1a484f7de013798e19b3f9e43e5889017cc5 +Author: James Baiera +Date: Mon Jun 3 14:58:06 2019 -0400 + + Enrich validate nested mappings (#42452) + + Ensures that fields retained in an enrich index from a source are not contained + under a nested field. It additionally makes sure that key fields exist, and that + value fields are checked if they are present. The policy runner test has also + been expanded with some faulty mapping test cases. + +commit 428beabc49739744cbad9ec09d5be2e6f0d00a00 +Author: Yu +Date: Tue Jun 4 00:37:59 2019 +0800 + + Remove "template" field in IndexTemplateMetaData (#42099) + + Remove "template" field from XContent parsing in IndexTemplateMetaData + +commit 00db9c1a2f1fc9718be7a613522b7317bc55c81e +Author: Armin Braun +Date: Mon Jun 3 19:29:36 2019 +0200 + + Make Connection Future Err. Handling more Resilient (#42781) (#42804) + + * There were a number of possible (runtime-) exceptions that could be raised in the adjusted code and prevent resolving the listener + * Relates #42350 + +commit 34fd9ce067272fc4668019d6def50acb7134aad4 +Author: Mark Vieira +Date: Mon Jun 3 10:25:52 2019 -0700 + + Fix error with test conventions on tasks that require Docker (#42719) + +commit 6c50246a58976f4f871413d2c83080fb30391267 +Author: Marios Trivyzas +Date: Mon Jun 3 17:50:49 2019 +0200 + + SQL: [Docs] Fix links syntax (#42806) + + Fix a couple of wrong links because of the order of the anchor + and the usage of backquotes. + + (cherry picked from commit 4e0c6525153b60a57202937c2ae57968c8e35285) + +commit b61202b0a8508e5a52a164036dc9c3ec862e1557 +Author: David Roberts +Date: Mon Jun 3 13:44:06 2019 +0100 + + [ML] Add a limit on line merging in find_file_structure (#42501) + + When analysing a semi-structured text file the + find_file_structure endpoint merges lines to form + multi-line messages using the assumption that the + first line in each message contains the timestamp. + However, if the timestamp is misdetected then this + can lead to excessive numbers of lines being merged + to form massive messages. + + This commit adds a line_merge_size_limit setting + (default 10000 characters) that halts the analysis + if a message bigger than this is created. This + prevents significant CPU time being spent subsequently + trying to determine the internal structure of the + huge bogus messages. + +commit 0253927ec45c9f3b3e0f4236f67f5e8e4426ce41 +Author: Benjamin Trent +Date: Mon Jun 3 06:53:44 2019 -0500 + + [ML Data Frame] Refactor stop logic (#42644) (#42763) + + * Revert "invalid test" + + This reverts commit 9dd8b52c13c716918ff97e6527aaf43aefc4695d. + + * Testing + + * mend + + * Revert "[ML Data Frame] Mute Data Frame tests" + + This reverts commit 5d837fa312b0e41a77a65462667a2d92d1114567. + + * Call onStop and onAbort outside atomic update + + * Don’t update CS + + * Tidying up + + * Remove invalid test that asserted logic that has been removed + + * Add stopped event + + * Revert "Add stopped event" + + This reverts commit 02ba992f4818bebd838e1c7678bd2e1cc090bfab. + + * Adding check for STOPPED in saveState + +commit 10aca87389b1260238e612474921788f79581076 +Author: David Roberts +Date: Mon Jun 3 12:18:14 2019 +0100 + + [ML] Better detection of binary input in find_file_structure (#42707) + + This change helps to prevent the situation where a binary + file uploaded to the find_file_structure endpoint is + detected as being text in the UTF-16 character set, and + then causes a large amount of CPU to be spent analysing + the bogus text structure. + + The approach is to check the distribution of zero bytes + between odd and even file positions, on the grounds that + UTF-16BE or UTF16-LE would have a very skewed distribution. + +commit bde0137e071e7b4b5684f3a6018098989a4efd64 +Author: Christoph Büscher +Date: Mon Jun 3 13:36:20 2019 +0200 + + Fix compile issue of earlier commit on 7.x + +commit 9a9ee9abedbfcaf49fd134349c60e11ab49f0859 +Author: Christoph Büscher +Date: Mon Jun 3 13:22:10 2019 +0200 + + [Docs] Add example to reimplement stempel analyzer (#42676) + + Adding an example of how to re-implement the polish stempel analyzer + in case a user want to modify or extend it. In order for the analyzer to be + able to use polish stopwords, also registering a polish_stop filter for the + stempel plugin. + + Closes #13150 + +commit df0f0b3d40589c3dca3d0f72a12c7e8458d84295 +Author: David Turner +Date: Mon Jun 3 12:12:07 2019 +0100 + + Rename autoMinMasterNodes to autoManageMasterNodes (#42789) + + Renames the `ClusterScope` attribute `autoMinMasterNodes` to reflect its + broader meaning since 7.0. + + Backport of the relevant part of #42700 to `7.x`. + +commit 2129d066430700176631893067d6f09babe324db +Author: Alan Woodward +Date: Mon Jun 3 09:16:54 2019 +0100 + + Create client-only AnalyzeRequest/AnalyzeResponse classes (#42197) + + This commit clones the existing AnalyzeRequest/AnalyzeResponse classes + to the high-level rest client, and adjusts request converters to use these new + classes. + + This is a prerequisite to removing the Streamable interface from the internal + server version of these classes. + +commit 929215c0d54c07bf0bd59f7ab66a08b250129977 +Author: Christian Kotzbauer +Date: Sat Jun 1 13:52:47 2019 +0200 + + Update release-notes.asciidoc (#42779) + +commit 9fdae169ac4ede7d94d17bfd28c55964c16dc021 +Author: Mark Vieira +Date: Fri May 31 16:34:16 2019 -0700 + + Use an anonymous inner class instead of lambda for UP-TO-DATE support + +commit 3a00d08c5031ea193c484e8401500e50cc9d161c +Author: Julie Tibshirani +Date: Fri May 31 08:53:59 2019 -0700 + + Clarify that inner_hits must be used to access nested fields. (#42724) + + This PR updates the docs for `docvalue_fields` and `stored_fields` to clarify + that nested fields must be accessed through `inner_hits`. It also tweaks the + nested fields documentation to make this point more visible. + + Addresses #23766. + +commit f51f8ed04ccf0e74d4273c36cb00e5f71cb4794b +Author: James Rodewig +Date: Fri May 31 13:03:41 2019 -0400 + + [DOCS] Remove unneeded options from `[source,sql]` code blocks (#42759) + + In AsciiDoc, `subs="attributes,callouts,macros"` options were required + to render `include-tagged::` in a code block. + + With elastic/docs#827, Elasticsearch Reference documentation migrated + from AsciiDoc to Asciidoctor. + + In Asciidoctor, the `subs="attributes,callouts,macros"` options are no + longer needed to render `include-tagged::` in a code block. This commit + removes those unneeded options. + + Resolves #41589 + +commit d0da30e5f44d77bd1a9923f2d7b8a04870b10750 +Author: Alan Woodward +Date: Fri May 31 13:14:18 2019 +0100 + + Return NO_INTERVALS rather than null from empty TokenStream (#42750) + + IntervalBuilder#analyzeText will currently return null if it is passed an + empty TokenStream, which can lead to a confusing NullPointerException + later on during querying. This commit changes the code to return + NO_INTERVALS instead. + + Fixes #42587 + +commit 61c6a26b3146916a49a24324a5efa06447a5b13a +Author: Jason Tedor +Date: Fri May 31 12:06:52 2019 -0400 + + Remove locale-dependent string checking + + We were checking if an exception was caused by a specific reason "Not a + directory". Alas, this reason is locale-dependent and can fail on + systems that are not set to en_US.UTF-8. This commit addresses this by + deriving what the locale-dependent error message would be and using that + for comparison with the actual exception thrown. + + Relates #41689 + +commit f22dcfb9dae79bd0689ea07a41a9b0c2a2172e06 +Author: Benjamin Trent +Date: Fri May 31 10:55:35 2019 -0500 + + [ML] [Data Frame] nesting group_by fields like other aggs (#42718) (#42760) + +commit 0a37dd7a869433d29f45379f49cab6baa20422b9 +Author: James Rodewig +Date: Fri May 31 11:08:54 2019 -0400 + + [DOCS] Remove unneeded `ifdef::asciidoctor[]` conditionals (#42758) + + Several `ifdef::asciidoctor` conditionals were added so that AsciiDoc + and Asciidoctor doc builds rendered consistently. + + With https://github.com/elastic/docs/pull/827, Elasticsearch Reference + documentation migrated completely to Asciidoctor. We no longer need to + support AsciiDoc so we can remove these conditionals. + + Resolves #41722 + +commit 478919c0bb7589a2c2c3f65380ca704182717300 +Author: James Rodewig +Date: Fri May 31 11:04:30 2019 -0400 + + [DOCS] Remove unneeded `ifdef::asciidoctor[]` conditionals (#42758) + + Several `ifdef::asciidoctor` conditionals were added so that AsciiDoc + and Asciidoctor doc builds rendered consistently. + + With https://github.com/elastic/docs/pull/827, Elasticsearch Reference + documentation migrated completely to Asciidoctor. We no longer need to + support AsciiDoc so we can remove these conditionals. + + Resolves #41722 + +commit 87ca762573cea2eab2ba4c7187630b46ac57cbde +Author: David Roberts +Date: Fri May 31 15:40:53 2019 +0100 + + [ML] Add Kibana application privilege to data frame admin/user roles (#42757) + + Data frame transforms are restricted by different roles to ML, but + share the ML UI. To prevent the ML UI being hidden for users who + only have the data frame admin or user role, it is necessary to add + the ML Kibana application privilege to the backend data frame roles. + +commit e687fd58fc547faac0725a04ee4d648d5d180d6b +Author: Jay Modi +Date: Fri May 31 08:03:10 2019 -0600 + + Re-enable token bwc tests (#42727) + + This commit re-enables token bwc tests that run as part of the rolling + upgrade tests. These tests were muted while #42651 was being + backported. + +commit d5061a151a3e303827226ab09f9b561643d44b69 +Author: Przemyslaw Gomulka +Date: Fri May 31 13:58:49 2019 +0200 + + Remove suppresions for "unchecked" for hamcrest varargs methods Backport(41528) #42749 + + In hamcrest 2.1 warnings for unchecked varargs were fixed by hamcrest using @SafeVarargs for those matchers where this warning occurred. + This PR is aimed to remove these annotations when Matchers.contains ,Matchers.containsInAnyOrder or Matchers.hasItems was used + backport #41528 + +commit 01446ff4bd7a389b694b7933cf3f2adf12a3260e +Author: Marios Trivyzas +Date: Fri May 31 12:56:07 2019 +0200 + + [Docs] Mention search related deprecations (#42751) + + Add deprecation entries for 7.3 regarding `common` query + and `cutoff_frequency` parameter. + + Follows: #42691 + +commit ab6b86bac920bf9f7a91903aefe7484bda3d7e07 +Author: Alan Woodward +Date: Fri May 31 11:23:36 2019 +0100 + + Add option to ObjectParser to consume unknown fields (#42491) + + ObjectParser has two ways of dealing with unknown fields: ignore them entirely, + or throw an error. Sometimes it can be useful instead to gather up these unknown + fields and record them separately, for example as arbitrary entries in a map. + + This commit adds the ability to specify an unknown field consumer on an ObjectParser, + called with the field name and parsed value of each unknown field encountered during + parsing. The public API of ObjectParser is largely unchanged, with a single new + constructor method and interface definition. + +commit 0ce7e7a4d840f148913455d58b9ef7b9575f4035 +Author: Przemyslaw Gomulka +Date: Fri May 31 09:42:32 2019 +0200 + + Enable tests failing due to java-joda warnings (#42693) + + Tests were failing in mixed cluster after more broad warnings were introduced + in 6.x These tests were using `yyyy-MM-dd` pattern which is now warning about + the change of `y` to `u`. However, using predefined pattern + `strict_date` which uses the same format prevents the warning from being + generate and allow smooth upgrade/work in mixed cluster. + relates #42679 + +commit f6779de2b7e66fffa2b12e4ddb0c2471ea783d2c +Author: Przemysław Witek +Date: Fri May 31 06:19:47 2019 +0200 + + Increase maximum forecast interval to 10 years. (#41082) (#42710) + + Increase the maximum duration to ~10 years (3650 days). + +commit 5f9382acc24d93c2e46e48b26669db62431763ea +Author: Alex Pang +Date: Thu May 30 18:01:04 2019 -0400 + + Fix docs typo in the certutil CSR mode (#42593) + + Changes the mention of `cert` to `csr`. + + Co-Authored-By: Alex Pang + +commit 215170b6c38b4230f8e82afa31baeec3719bcd19 +Merge: 77eed9e6a0c 47d50c67748 +Author: James Baiera +Date: Thu May 30 16:13:06 2019 -0400 + + Merge branch '7.x' into enrich-7.x + +commit 371cb9a8ce97b09e36b7139a307e324caee176f1 +Author: Jason Tedor +Date: Thu May 30 16:06:11 2019 -0400 + + Remove Log4j 1.2 API as a dependency (#42702) + + We had this as a dependency for legacy dependencies that still needed + the Log4j 1.2 API. This appears to no longer be necessary, so this + commit removes this artifact as a dependency. + + To remove this dependency, we had to fix a few places where we were + accidentally relying on Log4j 1.2 instead of Log4j 2 (easy to do, since + both APIs were on the compile-time classpath). + + Finally, we can remove our custom Netty logger factory. This was needed + when we were on Log4j 1.2 and handled logging in our own unique + way. When we migrated to Log4j 2 we could have dropped this + dependency. However, even then Netty would still pick up Log4j 1.2 since + it was on the classpath, thus the advantage to removing this as a + dependency now. + +commit d83b91d56a82ad76bc446cf9c0f725718f6a29c1 +Author: Lisa Cawley +Date: Thu May 30 12:12:37 2019 -0700 + + [DOCS] Disable Metricbeat system module (#42601) + +commit 1bb505c70dcd7a918bc4e96908a0946cc37310de +Author: Julie Tibshirani +Date: Thu May 30 09:23:38 2019 -0700 + + Clarify the settings around limiting nested mappings. (#42686) + + * Previously, we mentioned multiple times that each nested object was indexed as its own document. This is repetitive, and is also a bit confusing in the context of `index.mapping.nested_fields.limit`, as that applies to the number of distinct `nested` types in the mappings, not the number of nested objects. We now just describe the issue once at the beginning of the section, to illustrate why `nested` types can be expensive. + * Reference the ongoing example to clarify the meaning of the two settings. + + Addresses #28363. + +commit c1816354ed01d537c9a4f0608ab67aff2bb2cb9e +Author: Mark Vieira +Date: Thu May 30 10:29:42 2019 -0700 + + [Backport] Improve build configuration time (#42674) + +commit d14799f0a584d97696a46e5b1110ac88af21df8d +Author: David Turner +Date: Thu May 30 16:54:01 2019 +0100 + + Prevent merging nodes' data paths (#42665) + + Today Elasticsearch does not prevent you from reconfiguring a node's + `path.data` to point to data paths that previously belonged to more than one + node. There's no good reason to be able to do this, and the consequences can be + quietly disastrous. Furthermore, #42489 might result in a user trying to split + up a previously-shared collection of data paths by hand and there's definitely + scope for mixing the paths up across nodes when doing this. + + This change adds a check during startup to ensure that each data path belongs + to the same node. + +commit b5527b32781017f9be3e5a8dd4019b771186fc66 +Author: Benjamin Trent +Date: Thu May 30 12:05:35 2019 -0500 + + [ML] [Data Frame] add support for weighted_avg agg (#42646) (#42714) + +commit 711de2f59a785649c66f4a0d01e9fec29db35b46 +Author: Jay Modi +Date: Thu May 30 10:27:17 2019 -0600 + + Make hashed token ids url safe (#42651) + + This commit changes the way token ids are hashed so that the output is + url safe without requiring encoding. This follows the pattern that we + use for document ids that are autogenerated, see UUIDs and the + associated classes for additional details. + +commit ce30afcd01143946430bafcb568a65897101cb86 +Author: Marios Trivyzas +Date: Thu May 30 18:04:47 2019 +0200 + + Deprecate CommonTermsQuery and cutoff_frequency (#42619) (#42691) + + Since the max_score optimization landed in Elasticsearch 7, + the CommonTermsQuery is redundant and slower. Moreover the + cutoff_frequency parameter for MatchQuery and MultiMatchQuery + is redundant. + + Relates to #27096 + + (cherry picked from commit 04b74497314eeec076753a33b3b6cc11549646e8) + +commit ac8a9515a3c8bfb4ad769bc410ce1ebebd286002 +Author: Mark Vieira +Date: Thu May 30 08:38:17 2019 -0700 + + Remove usage of deprecated compare gradle builds plugin (#42687) + +commit bdd28cf4bbc76a0966f1319ef630a21f3ee36b3b +Author: Mark Vieira +Date: Thu May 30 08:37:58 2019 -0700 + + Remove unused Gradle plugin (#42684) + +commit 2a0c30c100034fcbcd68c495b26ce40643c3ef0d +Author: Mark Vieira +Date: Fri May 24 19:28:22 2019 -0700 + + Make JarHell task cacheable (#42551) + +commit 7cabe8acc9214634ea9a41d6d2f42064f0119bd6 +Author: Ioannis Kakavas +Date: Thu May 30 17:33:06 2019 +0300 + + Fix refresh remote JWKS logic (#42662) + + This change ensures that: + + - We only attempt to refresh the remote JWKS when there is a + signature related error only ( BadJWSException instead of the + geric BadJOSEException ) + - We do call OpenIDConnectAuthenticator#getUserClaims upon + successful refresh. + - We test this in OpenIdConnectAuthenticatorTests. + + Without this fix, when using the OpenID Connect realm with a remote + JWKSet configured in `op.jwks_path`, the refresh would be triggered + for most configuration errors ( i.e. wrong value for `op.issuer` ) + and the kibana wouldn't get a response and timeout since + `getUserClaims` wouldn't be called because + `ReloadableJWKSource#reloadAsync` wouldn't call `onResponse` on the + future. + +commit 5a76f46ac6850e18d5cdcf859096859a07518179 +Author: Mayya Sharipova +Date: Thu May 30 10:16:58 2019 -0400 + + Fix error with mapping in docs + + Related to #39630 + +commit b84b5525e1e615992e706c39b4d2d2d11e09bbb6 +Author: Peter Dyson +Date: Tue May 21 09:43:01 2019 +1000 + + [DOCS] path_hierarchy tokenizer examples (#39630) + + Closes #17138 + +commit 47d50c67748f67dec3e9c6e7d0757f767d0adc57 +Author: Armin Braun +Date: Thu May 30 14:55:55 2019 +0200 + + Fix Class Load Order in Netty4Plugin (#42591) (#42703) + + * Don't force the logger in the Netty4Plugin class already, at this point log4j might not be fully initialized. + * The call was redundant anyway since we do the same thing in the Netty4Transport and Netty4HttpServerTransport classes already and there we do it properly after setting up log4j by initilizing the loggers + * Relates #42532 + +commit 67326252d867dcd23c3ad54e261cee0bb92fc986 +Author: James Rodewig +Date: Thu May 30 08:30:30 2019 -0400 + + [DOCS] Rewrite 'wildcard' query (#42670) + +commit 1beed9e71f1cfe4db18b8fd4ff13a66622c386e6 +Author: Armin Braun +Date: Thu May 30 12:42:39 2019 +0200 + + Adjust use of Deprecated Netty API (#42613) (#42657) + + * With the recent upgrade to Netty 4.1.36 this method became deprecated and I made the advised change to fix the deprecation + +commit 86b1a07887d175dedb784d5e81a03506b4e86ac3 +Author: David Turner +Date: Thu May 30 08:02:13 2019 +0100 + + Log leader and handshake failures by default (#42342) + + Today the `LeaderChecker` and `HandshakingTransportAddressConnector` do not log + anything above `DEBUG` level. However there are some situations where it is + appropriate for them to log at a higher level: + + - if the low-level handshake succeeds but the high-level one fails then this + indicates a config error that the user should resolve, and the exception + will help them to do so. + + - if leader checks fail repeatedly then we restart discovery, and the exception + will help to determine what went wrong. + + Resolves #42153 + +commit 24a794fd6b5d3876eacf3dedab4819137cce2ee8 +Author: Ioannis Kakavas +Date: Thu May 30 07:43:00 2019 +0300 + + Fix testTokenExpiry flaky test (#42585) + + Test was using ClockMock#rewind passing the amount of nanoseconds + in order to "strip" nanos from the time value. This was intentional + as the expiration time of the UserToken doesn't have nanosecond + precision. + However, ClockMock#rewind doesn't support nanos either, so when it's + called with a TimeValue, it rewinds the clock by the TimeValue's + millis instead. This was causing the clock to go enough millis + before token expiration time and the test was passing. Once every + few hundred times though, the TimeValue by which we attempted to + rewind the clock only had nanos and no millis, so rewind moved the + clock back just a few millis, but still after expiration time. + + This change moves the clock explicitly to the same instant as expiration, + using clock.setTime and disregarding nanos. + +commit d2f9ccbe1856d4fadc9245b772d8326945bc7adb +Author: Igor Motov +Date: Wed May 29 20:05:12 2019 -0400 + + Geo: Refactor libs/geo parsers (#42549) + + Refactors the WKT and GeoJSON parsers from an utility class into an + instantiatable objects. This is a preliminary step in + preparation for moving out coordinate validators from Geometry + constructors. This should allow us to make validators plugable. + +commit e0dbf6e82aea0e804a21302f9a6e6800455c613f +Author: Gordon Brown +Date: Wed May 29 17:08:46 2019 -0600 + + Refactor HLRC RequestConverters parameters to be more explicit (#42128) + + The existing `RequestConverters.Params` is confusing, because it wraps + an underlying request object and mutations of the `Params` object + actually mutate the `Request` that was used in the construction of the + `Params`. + + This leads to a situation where we create a `RequestConverter.Params` + object, mutate it, and then it appears nothing happens to it - it + appears to be unused. What happens behind the scenes is that the Request + object is mutated when methods on `Params` are invoked. This results in + unclear, confusing code where mutating one object changes another with + no obvious connection. + + This commit refactors `RequestConverters.Params` to be a simple helper + class to produce a `Map` which must be passed explicitly to a Request + object. This makes it apparent that the `Params` are actually used, and + that they have an effect on the `request` object explicit and easier to + understand. + + Co-authored-by: Ojas Gulati + +commit 5e02dc6878d11534ff924555b0d4e15c5b54a3a1 +Author: Mayya Sharipova +Date: Wed May 29 16:09:36 2019 -0400 + + Add warning scores are floats (#42667) + +commit 95009963e32a2ea4175dc70f14d4bc03eedc05ae +Author: Jack Conradson +Date: Wed May 29 13:26:12 2019 -0700 + + Allow aggregations using expressions to use _score (#42652) + + _score was removed from use in aggregations using expressions + unintentionally when script contexts were added. This allows _score to once + again be used. + +commit 78f280de9c4045db46a7d3c89f1bd2241ef72c24 +Author: lcawl +Date: Wed May 29 11:00:47 2019 -0700 + + [DOCS] Adds more monitoring tagged regions + +commit 41208b7041d46f1580036091fd20540ec0113f32 +Author: James Baiera +Date: Wed May 29 13:54:21 2019 -0400 + + Muting prefilter shard tests as they are breaking in BWC testing + + See #42679 + +commit c5a410f68b9fe1d79f9a6757764bf8a2ae6dc4e3 +Author: David Kyle +Date: Tue May 28 15:58:03 2019 +0100 + + [ML Data Frame] Set DF task state when stopping (#42516) + + Set the state to stopped prior to persisting + +commit 3193dfa8e67df1defc9271b14d518c3d37902a88 +Author: James Rodewig +Date: Wed May 29 08:24:25 2019 -0400 + + [DOCS] Set explicit anchors for TLS/SSL settings (#42524) + +commit 7c612af6d242729889551445455beb51903e7382 +Author: kevin fuksman +Date: Wed May 29 08:44:24 2019 -0300 + + Added param ignore_throttled=false when indicesOptions.ignoreThrottled() is false (#42393) + + and fixed test RequestConvertersTests and added ignore_throttled on all request + +commit 345ff21ae504f3c6231b0fcc7da525afa8c3b055 +Author: Hendrik Muhs +Date: Wed May 29 11:13:37 2019 +0200 + + [ML-DataFrame] rewrite start and stop to answer with acknowledged (#42589) + + rewrite start and stop to answer with acknowledged + + fixes #42450 + +commit 53f5d313cd372d189ffe4ce3549e516abf9ef893 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed May 29 08:14:49 2019 +0200 + + Use correct global checkpoint sync interval (#42642) + + A disruption test case need to use a lower checkpoint sync interval + since they verify sequence numbers after the test waiting max 10 seconds + for it to stabilize. + + Closes #42637 + +commit 77eed9e6a0ccc428bfca4bfc7c1884afda2cc9bb +Author: Michael Basnight +Date: Tue May 28 14:47:50 2019 -0500 + + Add enrich policy GET API (#41384) + + This commit wires up the Rest calls and Transport calls for GET enrich + policy, as well as tests and rest spec additions. + +commit 8b325164f9c4e0f7d38e04c046491c7bd1c7df31 +Author: Julie Tibshirani +Date: Tue May 28 17:40:07 2019 -0700 + + Fix a callout in the field alias docs. + +commit be60125a4eadedd0bb9fbf48fce24eff53267bed +Merge: 484f5cee39a 6362ac14ad5 +Author: Michael Basnight +Date: Tue May 28 18:32:18 2019 -0500 + + Merge branch '7.x' into enrich-7.x + +commit 38f9e24411ff69533add6259f30c07666b9b5007 +Author: Adrien Grand +Date: Tue May 28 23:14:10 2019 +0200 + + Add 7.1.2 version constant. (#42648) + + Relates to #42635 + +commit 267e5a11104b4ced6ccafb958318e13e0ddd1646 +Author: Jim Ferenczi +Date: Tue May 28 22:11:34 2019 +0200 + + fix javadoc of SearchRequestBuilder#setTrackTotalHits (#42219) + +commit ace96a2b6eff1fe2098f93aac01240d99a900ea8 +Author: Hendrik Muhs +Date: Tue May 28 20:37:30 2019 +0200 + + check position before and after latch (#42623) + + check position before and after latch + + #fixes 42084 + +commit e54e74852a9157ac6e0585c1697c376e22554ac4 +Author: James Rodewig +Date: Tue May 28 15:18:51 2019 -0400 + + [DOCS] Fix X-Pack tag for Asciidoctor (#42443) + +commit 54d194409e256bfcd67d72bb17476c71493759c9 +Author: James Rodewig +Date: Tue May 28 14:20:42 2019 -0400 + + [DOCS] Set explicit anchors for Asciidoctor (#42521) + +commit ee1e4db266da389f9eb9aa8a33804b044141038e +Author: James Rodewig +Date: Tue May 28 14:16:18 2019 -0400 + + [DOCS] Set literal anchors for Asciidoctor (#42462) + +commit 6362ac14ad5159404c0d872c66a870b1a75ae9f7 +Author: Jason Tedor +Date: Tue May 28 13:04:19 2019 -0400 + + Lazily compute Java 8 home in reindex configuration (#42630) + + In the reindex from old tests we require Java 8. Today when configuring + the reindex from old tests, we eagerly evalulate Java 8 home, which + means that we require JAVA8_HOME to be set even if the reindex from old + test tasks are not in the task graph. This is an onerous requirement if, + for example, all that you want to do is build a distribution. This + commit addresses this by making evaluation of Java 8 home lazy, so that + it is only done and required if the reindex from old test tasks would be + executed. + +commit 77fc7b210767e2797c90df31f2bc169082108596 +Author: Lisa Cawley +Date: Tue May 28 09:04:02 2019 -0700 + + [DOCS] Reorg monitoring configuration for re-use (#42547) + +commit 8ff37e99f5590cabaf9fb33f5128b5a816b8d440 +Author: lcawl +Date: Tue May 28 08:52:45 2019 -0700 + + [DOCS] Removes coming tags + +commit 0e92ef1843a527f740598be7fa34fd6027e02408 +Author: Armin Braun +Date: Tue May 28 17:58:23 2019 +0200 + + Fix Incorrect Time Math in MockTransport (#42595) (#42617) + + * Fix Incorrect Time Math in MockTransport + + * The timeunit here must be nanos for the current time (we even convert it accordingly in the logging) + * Also, changed the log message when dumping stack traces a little to make it easier to grep for (otherwise it's the same as the message on unregister) + +commit 6166fed6f1fd51de02497bc0840be7909ac2647f +Author: Armin Braun +Date: Tue May 28 17:58:00 2019 +0200 + + Fix BulkProcessorRetryIT (#41700) (#42618) + + * Now that we process the bulk requests themselves on the WRITE threadpool, they can run out of retries too like the item requests even when backoff is active + * Fixes #41324 by using the same logic that checks failed item requests for their retry status for the top level bulk requests as well + +commit d06618a70d66bfe4a5c0f99fcd19ad16e8eebe90 +Author: Benjamin Trent +Date: Tue May 28 11:36:30 2019 -0400 + + [ML] adding delayed_data_check_config to datafeed update docs (#42095) (#42626) + + * [ML] adding delayed_data_check_config to datafeed update docs + + * [DOCS] Edits delayed data configuration details + +commit 130c832e10738308fed8a1c35321ef722dd521e4 +Author: Vigya Sharma +Date: Tue May 28 20:12:46 2019 +0530 + + Validate routing commands using updated routing state (#42066) + + When multiple commands are called in sequence, fetch shards + from mutable, up-to-date routing nodes to ensure each command's + changes are visible to subsequent commands. + + This addresses an issue uncovered during work on #41050. + +commit aea600fe7de17c1a718823e20f161cc081f899f0 +Author: David Kyle +Date: Tue May 28 09:15:03 2019 +0100 + + [Ml Data Frame] Return bad_request on preview when config is invalid (#42447) + +commit c21745c8abb9fd95193cc9c47dc581cb9ef2f52e +Author: David Turner +Date: Tue May 28 15:23:55 2019 +0100 + + Avoid loading retention leases while writing them (#42620) + + Resolves #41430. + +commit 31d2bdca37b7e9fc90241e31c85b695203e38c15 +Author: James Rodewig +Date: Tue May 28 08:56:50 2019 -0400 + + [DOCS] Fix Moving Avg Aggregation `deprecated` macro for Asciidoctor (#42405) + +commit b30ca8da28d8be127ad16c383587974b904b7bd7 +Author: James Rodewig +Date: Tue May 28 08:52:59 2019 -0400 + + [DOCS] Fix API Quick Reference rollup attribute for Asciidoctor (#42403) + +commit 3079d2d2951915d3465194ee2e3c525133db9dcd +Author: James Rodewig +Date: Tue May 28 08:47:18 2019 -0400 + + [DOCS] Escape cross-ref link comma for Asciidoctor (#42402) + +commit 1e0b0f640b41ce5f4f497fdf90b9dbfda271201c +Author: Yannick Welsch +Date: Tue May 28 13:56:13 2019 +0200 + + Fix compilation + + Follow-up to 5598647922c + +commit 5598647922cc439ed090caccff44cc8d1854d6ad +Author: Yannick Welsch +Date: Tue May 28 13:35:01 2019 +0200 + + Reset state recovery after successful recovery (#42576) + + The problem this commit addresses is that state recovery is not reset on a node that then becomes + master with a cluster state that has a state not recovered flag in it. The situation that was observed + in a failed test run of MinimumMasterNodesIT.testThreeNodesNoMasterBlock (see below) is that we + have 3 master nodes (node_t0, node_t1, node_t2), two of them are shut down (node_t2 remains), + when the first one comes back (renamed to node_t4) it becomes leader in term 2 and sends state + (with state_not_recovered_block) to node_t2, which accepts. node_t2 becomes leader in term 3, and + as it was previously leader in term1 and successfully completed state recovery, does never retry + state recovery in term 3. + + Closes #39172 + +commit 746a2f41fd8e27cbc7ad71ed4971679e99c97c9e +Author: David Turner +Date: Tue May 28 12:25:53 2019 +0100 + + Remove PRE_60_NODE_CHECKPOINT (#42531) + + This commit removes the obsolete `PRE_60_NODE_CHECKPOINT` constant for dealing + with 5.x nodes' lack of sequence number support. + + Backport of #42527 + +commit 00d665540a1cf06111e0c988289edd3e59d23b21 +Author: Armin Braun +Date: Tue May 28 12:44:40 2019 +0200 + + Make unwrapCorrupt Check Suppressed Ex. (#41889) (#42605) + + * Make unwrapCorrupt Check Suppressed Ex. (#41889) + * As discussed in #24800 we want to check for suppressed corruption + indicating exceptions here as well to more reliably categorize + corruption related exceptions + * Closes #24800, 41201 + +commit adb3574af830b04c2ce3b1b34d087c15ad9141db +Author: Daniel Mitterdorfer +Date: Tue May 28 12:25:18 2019 +0200 + + Mute NodeTests (#42615) + + Relates #42577 + Relates #42614 + +commit 116b050cc6ff04e5d992ae36f2817ff193959383 +Author: Armin Braun +Date: Tue May 28 11:00:28 2019 +0200 + + Cleanup Bulk Delete Exception Logging (#41693) (#42606) + + * Cleanup Bulk Delete Exception Logging + + * Follow up to #41368 + * Collect all failed blob deletes and add them to the exception message + * Remove logging of blob name list from caller exception logging + +commit 44bf784fe1a993fde79bb683208fa5508a7268cb +Author: Armin Braun +Date: Tue May 28 10:46:22 2019 +0200 + + Add Infrastructure to Run 3rd Party Repository Tests (#42586) (#42604) + + * Add Infrastructure to Run 3rd Party Repository Tests + + * Add infrastructure to run third party repository tests using our standard JUnit infrastructure + * This is a prerequisite of #42189 + +commit 635ce0ca6d2af56563bc91245e56035e93601803 +Author: Daniel Mitterdorfer +Date: Tue May 28 10:20:22 2019 +0200 + + Mute AsyncTwoPhaseIndexerTests#testStateMachine() (#42610) + + Relates #42084 + Relates #42609 + +commit 1d09367a82fb511a72d80dc0b7f7d37865e126ba +Author: Gürkan Kaymak +Date: Tue May 28 10:37:04 2019 +0300 + + Fixed ignoring name parameter for percolator queries (#42598) + + Closes #40405 + +commit 484f5cee39a3d57c248684192550aca38a4bbfb7 +Author: Martijn van Groningen +Date: Tue May 28 08:24:33 2019 +0200 + + Stricter update dependency between pipelines and components used by pipelines (#42038) + + Add support for components used by processor factories to get updated + before processor factories create new processor instances. + + Components can register via `IngestService#addIngestClusterStateListener(...)` + then if the internal representation of ingest pipelines get updated, + these components get updated with the current cluster state before + pipelines are updated. + + Registered EnrichProcessorFactory as ingest cluster state listener, so + that it has always an up to date view of the active enrich policies. + +commit c079fb61bfdfdc1ac2c8a61b9c61639ad1e4aa6e +Author: Armin Braun +Date: Tue May 28 08:00:02 2019 +0200 + + Remove Dead Code from Azure Repo Plugin (#42178) (#42569) + + * None of this stuff is used + +commit 2077f9ffbc8a4c7fd64884cd75147a7e546bd34e +Author: Nhat Nguyen +Date: Mon May 27 21:44:36 2019 -0400 + + Reset mock transport service in CcrRetentionLeaseIT (#42600) + + testRetentionLeaseIsAddedIfItDisappearsWhileFollowing does not reset the + mock transport service after test. Surviving transport interceptors from + that test can sneaky remove retention leases and make other tests fail. + + Closes #39331 + Closes #39509 + Closes #41428 + Closes #41679 + Closes #41737 + Closes #41756 + +commit ab832c4f17d198627354c80f4581f4bb53be95a0 +Author: Nhat Nguyen +Date: Mon May 27 20:38:58 2019 -0400 + + Use doc instead of _doc in FullClusterRestartIT + + ES does not accept doc type starting with underscore until 6.2.0. We + have to use "doc" instead of "_doc" in FullClusterRestartIT if we are + upgrading from a 6.2.0- cluster. + + Closes #42581 + +commit 4123ade2b6641cec674eb508ae86baabf0acea9a +Author: Nhat Nguyen +Date: Mon May 27 15:57:59 2019 -0400 + + Add test ensure we can execute update requests in mixed cluster + + Relates #42596 + +commit de6be819d66fb4ee081a9cad2de25231fdf05340 +Author: Nhat Nguyen +Date: Mon May 27 17:30:21 2019 -0400 + + Allocate to data-only nodes in ReopenWhileClosingIT (#42560) + + If all primary shards are allocated on the master node, then the + verifying before close step will never interact with mock transport + service. This change prefers to allocate shards on data-only nodes. + + Closes #39757 + +commit a94d24ae5ad2f7a1695cf83c2ad191c456c7dab4 +Author: Armin Braun +Date: Mon May 27 13:57:17 2019 +0200 + + Fix RareClusterStateIT (#42430) (#42580) + + * It looks like we might be cancelling a previous publication instead of + the one triggered by the given request with a very low likelihood. + * Fixed by adding a wait for no in-progress publications + * Also added debug logging that would've identified this problem + * Closes #36813 + +commit c4f44024af580840c060783f8c6bb7f1657484fd +Author: Armin Braun +Date: Mon May 27 12:24:20 2019 +0200 + + Remove Delete Method from BlobStore (#41619) (#42574) + + * Remove Delete Method from BlobStore (#41619) + * The delete method on the blob store was used almost nowhere and just duplicates the delete method on the blob containers + * The fact that it provided for some recursive delete logic (that did not behave the same way on all implementations) was not used and not properly tested either + +commit bb7e8eb2fd72cb9ed88d6e5932c46e3c526afa75 +Author: Armin Braun +Date: Mon May 27 12:08:45 2019 +0200 + + Introduce ShardState Enum + Slight Cleanup SnapshotsInProgress (#41940) (#42573) + + * Added separate enum for the state of each shard, it was really + confusing that we used the same enum for the state of the snapshot + overall and the state of each individual shard + * relates https://github.com/elastic/elasticsearch/pull/40943#issuecomment-488664150 + * Shortened some obvious spots in equals method and saved a few lines + via `computeIfAbsent` to make up for adding 50 new lines to this class + +commit a96606d9624da206cc82b2f6db89763d617433d2 +Author: Armin Braun +Date: Mon May 27 12:08:20 2019 +0200 + + Safer Wait for Snapshot Success in ClusterPrivilegeTests (#40943) (#42575) + + * Safer Wait for Snapshot Success in ClusterPrivilegeTests + + * The snapshot state returned by the API might become SUCCESS before it's fully removed from the cluster state. + * We should fix this race in the transport API but it's not trivial and will be part of the incoming big round of refactoring the repository interaction, this added check fixes the test for now + * closes #38030 + +commit 5d3e381648e697e1403e92eefae03f843e2605d0 +Author: Ignacio Vera +Date: Mon May 27 12:05:17 2019 +0200 + + mute test testClosedIndices (#42582) + +commit 381e100217d92f031a58f4052b0be81519c46984 +Author: Travis Steel +Date: Mon May 27 04:53:51 2019 -0500 + + Fixed typo in docker.asciidoc (#42455) + +commit 380f296631f49721ab85eac21946d3729087ac00 +Author: bellengao +Date: Mon May 27 17:47:46 2019 +0800 + + Update script-fields.asciidoc (#42490) + +commit 7b4d1ac35252e13f4856d46399b94159a05c702c +Author: Armin Braun +Date: Mon May 27 11:47:04 2019 +0200 + + Remove Obsolete BwC Logic from BlobStoreRepository (#42193) (#42571) + + * Remove Obsolete BwC Logic from BlobStoreRepository + + * We can't restore 1.3.3 files anyway -> no point in doing the dance of computing a hash here + * Some other minor+obvious cleanups + +commit b68358945f2d3818bf9d7fae80923520b857cde2 +Author: Armin Braun +Date: Mon May 27 11:44:36 2019 +0200 + + Dump Stacktrace on Slow IO-Thread Operations (#42000) (#42572) + + * Dump Stacktrace on Slow IO-Thread Operations + + * Follow up to #39729 extending the functionality to actually dump the + stack when the thread is blocked not afterwards + * Logging the stacktrace after the thread became unblocked is only of + limited use because we don't know what happened in the slow callback + from that (only whether we were blocked on a read,write,connect etc.) + * Relates #41745 + +commit 489616da6255c830f69aa36e6117011081db0503 +Author: Armin Braun +Date: Mon May 27 11:39:59 2019 +0200 + + Fix testTracerLog Network Tests (#42286) (#42565) + + * Fix testTracerLog Network Tests + + * Start appender before using it like we do for e.g. the Netty leak detection appender to avoid interference from actions on the network threads that might still be dangling from previous tests in the same suite + * Closes #41890 + +commit c7448b12e1e1c995929d25154ea591bb77f96802 +Author: Armin Braun +Date: Mon May 27 11:28:50 2019 +0200 + + Cleanup Redundant BlobStoreFormat Class (#42195) (#42570) + + * No need to have an abstract class here when there's only a single impl. + +commit 49767fc1e9b4d9fead55aac0c47df1b41565aca4 +Author: Armin Braun +Date: Mon May 27 11:28:12 2019 +0200 + + Some Cleanup in o.e.gateway Package (#42108) (#42568) + + * Removing obvious dead code + * Removing redundant listener interface + +commit a5ca20a2506328f7068736cb7a447ef83ceb51e4 +Author: Armin Braun +Date: Mon May 27 11:04:54 2019 +0200 + + Some Cleanup in o.e.i.engine (#42278) (#42566) + + * Some Cleanup in o.e.i.engine + + * Remove dead code and parameters + * Reduce visibility in some obvious spots + * Add missing `assert`s (not that important here since the methods + themselves will probably be dead-code eliminated) but still + +commit d2cd36bd9f96645a76a99eea2963dc758d1c32d5 +Author: Armin Braun +Date: Mon May 27 10:38:03 2019 +0200 + + Upgrade to Netty 4.1.36 (#42543) (#42564) + +commit a91cec4c46eb22e0bfb038023210be12f4d690af +Merge: 59012857733 e591d30918f +Author: Martijn van Groningen +Date: Mon May 27 10:18:02 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit e591d30918f489d3aac79abd825fba5ae3061828 +Author: Martijn van Groningen +Date: Mon May 27 10:17:00 2019 +0200 + + fixed test compile issue + +commit 59012857733fa60550d898955e0bfac625a9e51b +Author: Martijn van Groningen +Date: Fri May 24 08:46:50 2019 +0200 + + Complete EnrichIT by using the execute enrich policy API (#42433) + +commit 48a71459c0a6bca5885cfea32179a315e4b66758 +Author: Martijn van Groningen +Date: Mon May 27 09:21:30 2019 +0200 + + Improve how internal representation of pipelines are updated (#42257) + + If a single pipeline is updated then the internal representation of + all pipelines was updated. With this change, only the internal representation + of the pipelines that have been modified will be updated. + + Prior to this change the IngestMetadata of the previous and current cluster + was used to determine whether the internal representation of pipelines + should be updated. If applying the previous cluster state change failed then + subsequent cluster state changes that have no changes to IngestMetadata + will not attempt to update the internal representation of the pipelines. + + This commit, changes how the IngestService updates the internal representation + by keeping track of the underlying configuration and use that to detect + against the new IngestMetadata whether a pipeline configuration has been + changed and if so, then the internal pipeline representation will be updated. + +commit 4dbf6c0df9cb84d8b22b77dd7cde66cc6453ee4b +Author: Alpar Torok +Date: Wed May 22 22:00:51 2019 +0300 + + Make packer cache branches explicit (#41990) + + Before this change we would recurse to cache bwc versions. + This proved to be problematic due to the number of steps it was + generating taking too long. + Also this required tricky maintenance to break the recursion for old + branches we don't really care about. + + With this change we now cache specific branches only. + +commit 85e60850af712c30f921b2a6e7dca9be3690f2e5 +Author: Nhat Nguyen +Date: Sun May 26 16:03:42 2019 -0400 + + Add debug log for retention leases (#42557) + + We need more information to understand why CcrRetentionLeaseIT is + failing. This commit adds some debug log to retention leases and enables + them in CcrRetentionLeaseIT. + +commit 5d2fcc53e4dde4632244858fb9f00cb749d5afba +Author: Nhat Nguyen +Date: Sun May 26 11:18:39 2019 -0400 + + Unmute FullClusterRestartIT#testClosedIndices + + Fixed in #39566 + Closes #39576 + +commit d6e2f4a43e8d91862dd2f0cdc68de16a04ed9e20 +Author: Nhat Nguyen +Date: Fri May 24 22:13:48 2019 -0400 + + Enable recoveries trace log in CcrRetentionLeaseIT + + Tracked #41679 + +commit 6bec8766828070a640f5ad1a7dac4a661b11607c +Author: Tanguy Leroux +Date: Thu May 23 20:18:34 2019 +0200 + + Improve Close Index Response (#39687) + + This changes the `CloseIndexResponse` so that it reports closing result + for each index. Shard failures or exception are also reported per index, + and the global acknowledgment flag is computed from the index results + only. + + The response looks like: + ``` + { + "acknowledged" : true, + "shards_acknowledged" : true, + "indices" : { + "docs" : { + "closed" : true + } + } + } + ``` + + The response reports shard failures like: + ``` + { + "acknowledged" : false, + "shards_acknowledged" : false, + "indices" : { + "docs-1" : { + "closed" : true + }, + "docs-2" : { + "closed" : false, + "shards" : { + "1" : { + "failures" : [ + { + "shard" : 1, + "index" : "docs-2", + "status" : "BAD_REQUEST", + "reason" : { + "type" : "index_closed_exception", + "reason" : "closed", + "index_uuid" : "JFmQwr_aSPiZbkAH_KEF7A", + "index" : "docs-2" + } + } + ] + } + } + }, + "docs-3" : { + "closed" : true + } + } + } + ``` + + Co-authored-by: Tanguy Leroux + +commit 63eccb16de71b44fe8234c9d45b0c0861a4ae6d6 +Author: Mark Vieira +Date: Fri May 24 18:29:40 2019 -0700 + + Make LoggerUsageTask cacheable (#42550) + + (cherry picked from commit 0bb46d73cb89016ab9d96e76693bb0d7cee267a1) + +commit 24cf86a01334cb4dc9b66e20d249e03e8bbed347 +Author: Mark Vieira +Date: Fri May 24 18:08:51 2019 -0700 + + Ignore JAR manifests when snapshotting runtime classpaths (#42548) + + (cherry picked from commit d5281fc96f6fb2f022c87699bdad64d88614e04c) + +commit 3a6c2525ca6d72f7128ceb8528804026c2aa8e8a +Author: Julie Tibshirani +Date: Fri May 24 15:55:06 2019 -0700 + + Deprecate support for chained multi-fields. (#42330) + + This PR contains a straight backport of #41926, and also updates the + migration documentation and deprecation info API for 7.x. + +commit f2cfd09289d9fd77bf88b108cef6629dd4cb8e91 +Author: Jason Tedor +Date: Fri May 24 15:39:27 2019 -0700 + + Remove renewal in retention lease recovery test (#42536) + + This commit removes the act of renewing some retention leases during a + retention lease recovery test. Having renewal does not add anything + extra to this test, but does allow for some situations where the test + can fail spuriously (i.e., in a way that does not indicate that + production code is broken). + +commit 5695992094641ce63cad34a5c4384876e623b0f2 +Author: Mark Vieira +Date: Fri May 24 13:46:59 2019 -0700 + + Use reproducible method of generating properties file for better caching (#42539) + + (cherry picked from commit 9772574f9d0b942a1ee8dba5ff503b4cd286e36c) + +commit 74d771d8f69d3d743b521ac4cecfb7544b41d8bc +Author: Nhat Nguyen +Date: Fri May 24 15:42:59 2019 -0400 + + Adjust load SplitIndexIT#testSplitIndexPrimaryTerm (#42477) + + SplitIndexIT#testSplitIndexPrimaryTerm sometimes timeout due to + relocating many shards. This change adjusts loads and increases + the timeout. + +commit 02739d038ccbbb25a2c26096a75fb2a5de196e85 +Author: Nhat Nguyen +Date: Thu May 23 13:27:34 2019 -0400 + + Mute accounting circuit breaker check after test (#42448) + + If we close an engine while a refresh is happening, then we might leak + refCount of some SegmentReaders. We need to skip the ram accounting + circuit breaker check until we have a new Lucene snapshot which includes + the fix for LUCENE-8809. + + This also adds a test to the engine but left it muted so we won't forget + to reenable this check. + + Closes #30290 + +commit 48dc0dca570be0f0442abb8c43b7d3907489e09a +Author: David Roberts +Date: Fri May 24 20:05:15 2019 +0100 + + [ML] Use map and filter instead of flatMap in find_file_structure (#42534) + + Using map and filter avoids the garbage from all the + Stream.of calls that flatMap necessitated. Performance + is better when there are masses of fields. + +commit 34de68b007ed8e2051f9adf9531faf1b6b69dcb4 +Author: David Roberts +Date: Fri May 24 20:04:05 2019 +0100 + + [ML] Fix possible race condition when closing an opening job (#42506) + + This change fixes a race condition that would result in an + in-memory data structure becoming out-of-sync with persistent + tasks in cluster state. + + If repeated often enough this could result in it being + impossible to open any ML jobs on the affected node, as the + master node would think the node had capacity to open another + job but the chosen node would error during the open sequence + due to its in-memory data structure being full. + + The race could be triggered by opening a job and then closing + it a tiny fraction of a second later. It is unlikely a user + of the UI could open and close the job that fast, but a script + or program calling the REST API could. + + The nasty thing is, from the externally observable states and + stats everything would appear to be fine - the fast open then + close sequence would appear to leave the job in the closed + state. It's only later that the leftovers in the in-memory + data structure might build up and cause a problem. + +commit d521a88e19d5c6db80c1fe2c998d1e72576c4da2 +Author: James Rodewig +Date: Fri May 24 15:03:11 2019 -0400 + + [DOCS] Move callouts to end of line for Asciidoctor migration (#42356) + +commit 6d47ee92684d4c17eeb2862eda3d60d5fc53d7bf +Author: Hendrik Muhs +Date: Fri May 24 19:54:06 2019 +0200 + + [ML-DataFrame] add support for fixed_interval, calendar_interval, remove interval (#42427) + + * add support for fixed_interval, calendar_interval, remove interval + + * adapt HLRC + + * checkstyle + + * add a hlrc to server test + + * adapt yml test + + * improve naming and doc + + * improve interface and add test code for hlrc to server + + * address review comments + + * repair merge conflict + + * fix date patterns + + * address review comments + + * remove assert for warning + + * improve exception message + + * use constants + +commit e28a9e99c403ff7dd68e637fedee7f45e4fde944 +Author: Igor Motov +Date: Fri May 24 13:16:48 2019 -0400 + + SQL: Moves the JTS-based tests suppression to Before (#42526) + + Moves the test suppression from `ClassRule` to `Before`, where it is + properly handled in the CI build. + + Fixes #42221 + +commit 09e8910b0feaf72dd382dc6b96c012886e7e2c79 +Author: David Roberts +Date: Fri May 24 18:44:51 2019 +0100 + + [DOCS] Adding ML-specific prerequisites to setup docs (#42529) + +commit 43dd081e223cdaf0bb52b797d35873b6a21545a6 +Author: James Rodewig +Date: Fri May 24 13:39:29 2019 -0400 + + [DOCS] Fix nested def list for Asciidoctor (#42353) + +commit 329d1307a59a01914ca945010c533ce4f59a7d5c +Author: Nhat Nguyen +Date: Fri May 24 11:19:41 2019 -0400 + + Add test to verify force primary allocation on closed indices (#42458) + + This change adds a test verifying that we can force primary allocation on closed indices. + +commit 075fd2a0acf95a892e4fdf667e95df4af386f5e2 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri May 24 16:48:45 2019 +0200 + + Shard CLI tool always check shards (#41480) + + The shard CLI tool would not do anything if a corruption marker was not + present. But a corruption marker is only added if a corruption is + detected during indexing/writing, not if a search or other read fails. + + Changed the tool to always check shards regardless of corruption marker + presence. + + Related to #41298 + +commit 2325ffb757a5b5b0cb6da35dc52a612cb49f08e6 +Author: Michael Basnight +Date: Wed May 22 11:19:27 2019 -0500 + + Add enrich policy execute API (#41762) + + This commit wires up the Rest calls and Transport calls for execute + enrich policy, as well as tests and rest spec additions. + +commit 523b5bfdb556a4fa01c9123f3a4e4c6157675ff1 +Author: Marios Trivyzas +Date: Fri May 24 15:20:14 2019 +0200 + + Fix sorting on nested field with unmapped (#42451) + + Previously sorting on a missing nested field would fail with an + Exception: + `[nested_field] failed to find nested object under path [nested_path]` + despite `unmapped_type` being set on the query. + + Fixes: #33644 + + (cherry picked from commit 631142d5dd088a10de8dcd939b50a14301173283) + +commit 12d5642e934c9a99ad8b01662fe68a838d6478bc +Author: Christoph Büscher +Date: Fri May 24 15:27:10 2019 +0200 + + Small internal AnalysisRegistry changes (#42500) + + Some internal refactorings to the AnalysisRegistry, spin-off from #40782. + +commit a5b6ed8d1efb8d249fd053e1388978736f697a60 +Author: David Turner +Date: Fri May 24 14:26:49 2019 +0100 + + Remove AwaitsFix of #41967 following #42504 + +commit 4d02ca1633ef832e9dab691b6e6be0d5a9c575ac +Author: David Turner +Date: Fri May 24 14:17:21 2019 +0100 + + Drain master task queue when stabilising (#42504) + + Today the default stabilisation time is calculated on the assumption that the + elected master has no pending tasks to process when it is elected, but this is + not a safe assumption to make. This can result in a cluster reaching the end of + its stabilisation time without having stabilised. Furthermore in #36943 we + increased the probability that each step in `runRandomly()` enqueues another + task, vastly increasing the chance that we hit such a situation. + + This change extends the stabilisation process to allow time for all pending + tasks, plus a task that might currently be in flight. + + Fixes #41967, in which the master entered the stabilisation phase with over 800 + tasks to process. + +commit 56677f69cf34e26912a5e9e7aca7d6f7e80c7f37 +Author: Marios Trivyzas +Date: Fri May 24 14:03:05 2019 +0200 + + Mute testTrustConfigReloadsFileContents + + Tracked by #42509 + +commit 40348ab726125a3d7b336ae0205ddf34adf2c6ce +Author: weizijun +Date: Wed Feb 27 00:58:35 2019 +0800 + + Use accurate total hits in IndexPrimaryRelocationIT + + By default, we track total hits up to 10k but we might index more than + 10k documents `testPrimaryRelocationWhileIndexing`. With this change, we + always request for the accurate total hits in the test. + + > java.lang.AssertionError: Count is 10000+ hits but 11684 was expected. + +commit 46ccfba8087e7f2a4c64be812da8762aa9e43c30 +Author: Simon Willnauer +Date: Fri May 24 10:44:59 2019 +0200 + + Remove IndexStore and DirectoryService (#42446) + + Both of these classes are basically a bloated wrapper around a simple + construct that can simply be a DirectoryFactory interface. This change + removes both classes and replaces them with a simple stateless interface + that creates a new `Directory` per shard. The concept of `index.store` is preserved + since it makes sense from a configuration perspective. + +commit 79fa7d809865976397d40c192a16cbfb91130f2a +Merge: 824ccfabd9a 7cee294acfe +Author: Martijn van Groningen +Date: Fri May 24 11:44:35 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 7cee294acfefe997ef98905e9dfd85af3726725c +Author: Hendrik Muhs +Date: Fri May 24 11:05:30 2019 +0200 + + [ML-DataFrame]backport dataframe changes from 42202, using client instead of transport (#42468) + + backport dataframe changes from #42202, using client instead of transport + +commit f472186b9fc98494efe28e74a7c710d3c42b09ab +Author: David Roberts +Date: Thu May 23 21:06:47 2019 +0100 + + [ML] Improve file structure finder timestamp format determination (#41948) + + This change contains a major refactoring of the timestamp + format determination code used by the ML find file structure + endpoint. + + Previously timestamp format determination was done separately + for each piece of text supplied to the timestamp format finder. + This had the drawback that it was not possible to distinguish + dd/MM and MM/dd in the case where both numbers were 12 or less. + In order to do this sensibly it is best to look across all the + available timestamps and see if one of the numbers is greater + than 12 in any of them. This necessitates making the timestamp + format finder an instantiable class that can accumulate evidence + over time. + + Another problem with the previous approach was that it was only + possible to override the timestamp format to one of a limited + set of timestamp formats. There was no way out if a file to be + analysed had a timestamp that was sane yet not in the supported + set. This is now changed to allow any timestamp format that can + be parsed by a combination of these Java date/time formats: + yy, yyyy, M, MM, MMM, MMMM, d, dd, EEE, EEEE, H, HH, h, mm, ss, + a, XX, XXX, zzz + Additionally S letter groups (fractional seconds) are supported + providing they occur after ss and separated from the ss by a dot, + comma or colon. Spacing and punctuation is also permitted with + the exception of the question mark, newline and carriage return + characters, together with literal text enclosed in single quotes. + + The full list of changes/improvements in this refactor is: + + - Make TimestampFormatFinder an instantiable class + - Overrides must be specified in Java date/time format - Joda + format is no longer accepted + - Joda timestamp formats in outputs are now derived from the + determined or overridden Java timestamp formats, not stored + separately + - Functionality for determining the "best" timestamp format in + a set of lines has been moved from TextLogFileStructureFinder + to TimestampFormatFinder, taking advantage of the fact that + TimestampFormatFinder is now an instantiable class with state + - The functionality to quickly rule out some possible Grok + patterns when looking for timestamp formats has been changed + from using simple regular expressions to the much faster + approach of using the Shift-And method of sub-string search, + but using an "alphabet" consisting of just 1 (representing any + digit) and 0 (representing non-digits) + - Timestamp format overrides are now much more flexible + - Timestamp format overrides that do not correspond to a built-in + Grok pattern are mapped to a %{CUSTOM_TIMESTAMP} Grok pattern + whose definition is included within the date processor in the + ingest pipeline + - Grok patterns that correspond to multiple Java date/time + patterns are now handled better - the Grok pattern is accepted + as matching broadly, and the required set of Java date/time + patterns is built up considering all observed samples + - As a result of the more flexible acceptance of Grok patterns, + when looking for the "best" timestamp in a set of lines + timestamps are considered different if they are preceded by + a different sequence of punctuation characters (to prevent + timestamps far into some lines being considered similar to + timestamps near the beginning of other lines) + - Out-of-the-box Grok patterns that are considered now include + %{DATE} and %{DATESTAMP}, which have indeterminate day/month + ordering + - The order of day/month in formats with indeterminate day/month + order is determined by considering all observed samples (plus + the server locale if the observed samples still do not suggest + an ordering) + + Relates #38086 + Closes #35137 + Closes #35132 + +commit f864f6a7405dffd587a6ad82cee445f61df6fec1 +Author: David Turner +Date: Fri May 24 08:43:16 2019 +0100 + + Cluster state from API should always have a master (#42454) + + Today the `TransportClusterStateAction` ignores the state passed by the + `TransportMasterNodeAction` and obtains its state from the cluster applier. + This might be inconsistent, showing a different node as the master or maybe + even having no master. + + This change adjusts the action to use the passed-in state directly, and adds + tests showing that the state returned is consistent with our expectations even + if there is a concurrent master failover. + + Fixes #38331 + Relates #38432 + +commit 528f8cc073a5daaa3bbb4068f5095579f34ed665 +Author: David Turner +Date: Fri May 24 08:37:22 2019 +0100 + + Add stack traces to RetentionLeasesIT failures (#42425) + + Today `RetentionLeaseIT` calls `fail(e.toString())` on some exceptions, losing + the stack trace that came with the exception. This commit adjusts this to + re-throw the exception wrapped in an `AssertionError` so we can see more + details about failures such as #41430. + +commit c0974a9813385fe6589005ea73c3185a6f1848c5 +Author: David Turner +Date: Fri May 24 08:27:06 2019 +0100 + + Add more logging to MockDiskUsagesIT (#42424) + + This commit adds a log message containing the routing table, emitted on each + iteration of the failing assertBusy() in #40174. It also modernizes the code a + bit. + +commit f3c33d6d9665fdc20af6e7f2270acaea4c0de266 +Author: Adrien Grand +Date: Fri May 24 09:25:24 2019 +0200 + + Add 7.1.1 release notes. + +commit 200b7e1888472bde50b00cc395e180dd756a3ae3 +Author: Mark Vieira +Date: Thu May 23 18:17:57 2019 -0700 + + Gradle init script for enabling remote build cache + +commit 567c0d331f1b9c1bbd7f51f12cf79c056f06b67d +Author: Tim Vernum +Date: Fri May 24 13:16:26 2019 +1000 + + Fix settings prefix for realm truststore password (#42413) + + As part of #30241 realm settings were changed to be true affix + settings. In the process of this change, the "ssl." prefix was lost + from the realm truststore password. It should be: + + xpack.security.authc.realms...ssl.truststore.password + + Due to a mismatch between the way we define SSL settings and load SSL + contexts, there was no way to define this legacy password setting in a + realm config. + + The settings validation would reject "ssl.truststore.password" but the + SSL service would ignore "truststore.password" + + Backport of: #42336 + +commit 167f391cfdac10c4022759b7d3583022475f34a8 +Author: Jack Conradson +Date: Thu May 23 18:59:30 2019 -0400 + + Bug fix to allow access to top level params in reduce script (#42096) + +commit a49bafc194aebb505513934211fe5dd4ffc49700 +Author: Ryan Ernst +Date: Thu May 23 14:01:07 2019 -0700 + + Split document and metadata fields in GetResult (#38373) (#42456) + + This commit makes creators of GetField split the fields into document fields and metadata fields. It is part of larger refactoring that aims to remove the calls to static methods of MapperService related to metadata fields, as discussed in #24422. + +commit 14f29de2a8eb01cec34c6d16be4293f8f5099f0a +Author: David Roberts +Date: Thu May 23 21:04:03 2019 +0100 + + Avoid HashMap construction on Grok non-match (#42444) + + This change moves the construction of the result + HashMap in Grok.captures() into the branch that + actually needs it. + + This probably will not make a measurable difference + for ingest pipelines, but it is beneficial to the + ML find_file_structure endpoint, as it tries out + many Grok patterns that will fail to match. + +commit 2b22ceac044fb4cc8e469f0ab31e342722b815e3 +Author: Jake Landis +Date: Thu May 23 14:22:16 2019 -0500 + + Bulk processor concurrent requests (#41451) (#42438) + + `org.elasticsearch.action.bulk.BulkProcessor` is a threadsafe class that + allows for simple semantics to deal with sending bulk requests. Once a + bulk reaches it's pre-defined size, documents, or flush interval it will + execute sending the bulk. One configurable option is the number of concurrent + outstanding bulk requests. That concurrency is implemented in + `org.elasticsearch.action.bulk.BulkRequestHandler` via a semaphore. However, + the only code that currently calls into this code is blocked by `synchronized` + methods. This results in the in-ability for the BulkProcessor to behave concurrently + despite supporting configurable amounts of concurrent requests. + + This change removes the `synchronized` method in favor an explicit + lock around the non-thread safe parts of the method. The call into + `org.elasticsearch.action.bulk.BulkRequestHandler` is no longer blocking, which + allows `org.elasticsearch.action.bulk.BulkRequestHandler` to handle it's own concurrency. + +commit a48125a9f7d1fcfaab2e240f79d7533c783c3fc5 +Author: Costin Leau +Date: Thu May 23 21:30:41 2019 +0300 + + Fix FROZEN indices backport + +commit 9fdf4215dd0f941dce74a708b532900b85228210 +Author: Costin Leau +Date: Fri May 10 14:19:26 2019 +0300 + + Docs: Documentation for the upcoming SQL support of frozen indices (#41863) + + (cherry picked from commit a3cc03eb1503df24c1706a721fcc9af38c3b2873) + (cherry picked from commit f42dcf2ffd7bd25f3f91aa6127515f393cd1860f) + +commit d5f04d29c99b5d39078b3de99b2b2e1bd8a93bfe +Author: Costin Leau +Date: Fri May 10 00:26:56 2019 +0300 + + SQL: Add support for FROZEN indices (#41558) + + Allow querying of FROZEN indices both through dedicated SQL grammar + extension: + > SELECT field FROM FROZEN index + and also through driver configuration parameter, namely: + > index.include.frozen: true/false + + Fix #39390 + Fix #39377 + + (cherry picked from commit 2445a933915f420c7f51e8505afa0a7978ce6b0f) + +commit a142e8cfd8408b96b2df4bd7d5d84aa141710493 +Author: emasab +Date: Thu May 23 18:35:39 2019 +0200 + + Build local year inside DateFormat lambda + + bugfix for https://github.com/elastic/elasticsearch/issues/41797 (#42120) + + This makes sure that the year can change between when the lambda is generated and when it is executed without causing the incorrect year to be used. + + Resolves #41797 + +commit c9ed91312a766d5b08e11b659f7cb55425b3a799 +Author: Mark Vieira +Date: Thu May 23 09:16:34 2019 -0700 + + Mute slow and flaky build-tools integration tests + +commit 5a884dac03299b2d62d084a30bea48d000478ff6 +Author: Simon Willnauer +Date: Thu May 23 17:08:29 2019 +0200 + + Unguice Snapshot / Restore services (#42357) + + This removes the @Inject annotations from the Snapshot/Restore infrastructure + classes and registers them manually in Node.java + +commit a49760321934175f7e8c05432f18eeb90df62fee +Author: Jim Ferenczi +Date: Thu Apr 25 21:09:32 2019 +0200 + + Disable max score optimization for queries with unbounded max scores (#41361) + + Lucene 8 has the ability to skip blocks of non-competitive documents. + However some queries don't track their maximum score (`script_score`, `span`, ...) + so they always return Float.POSITIVE_INFINITY as maximum score. This can slow down + some boolean queries if other clauses have bounded max scores. This commit disables + the max score optimization when we detect a mandatory scoring clause with unbounded + max scores. Optional clauses are not checked since they can still skip documents + when the unbounded clause is after the current document. + +commit f57fdc57e9e194ef3ff4aa40e7da90c28e449165 +Author: Yannick Welsch +Date: Thu May 23 15:59:55 2019 +0200 + + Deprecate max_local_storage_nodes (#42426) + + Allows this setting to be removed in 8.0, see #42428 + +commit 85ff9543b73806e8624ca7e8e78f977dd175555a +Author: Christoph Büscher +Date: Thu May 23 09:53:16 2019 -0400 + + Prevent normalizer from not being closed on exception (#42375) + + Currently AnalysisRegistry#processNormalizerFactory creates a normalizer and + only later checks whether it should be added to the normalizer map passed in. In + case we throw an exception it isn't closed. This can be prevented by moving the + check that throws the exception earlier. + +commit 6d8a0e36ec496f3f9861fcddc46eaf2622735cb6 +Author: Zachary Tong +Date: Tue May 21 10:25:23 2019 -0400 + + Re-mute all ml_datafeed_crud rolling upgrade tests + + AwaitsFix https://github.com/elastic/elasticsearch/issues/42258 + + Thought this was fixed, but throwing deprecation warnings at + an unexpected time so putting this back on mute until we + figure it out. + +commit a23257ce06625db8fa266c0cb00ff32c8b92277f +Author: David Kyle +Date: Thu May 23 08:42:06 2019 +0100 + + [ML Data Frame] Account for completed data frames in test (#42351) + + When asserting on the checkpoint value if the DF has completed the checkpoint will be 1 else 0. + Similarly state may be started or indexing. Closes #42309 + +commit c2c8d0e63727e577f300ef17bffd44fe929d6e28 +Author: markharwood +Date: Thu May 23 10:34:13 2019 +0100 + + Test fix - results equality failed because of subtle scoring differences between replicas. (#42366) + + Diverging merge policies means the segments and therefore scores are not the same. + Fixed the test by ensuring there are zero replicas. + + Closes #32492 + +commit b88e80ab89602ac4d1903c7465deca45d8c04e84 +Author: Jim Ferenczi +Date: Thu May 23 11:41:05 2019 +0200 + + Upgrade to Lucene 8.1.0 (#42214) + + This commit upgrades to the GA release of Lucene 8.1.0 + +commit 4ca5649a0d6c9a823eb139d84202c9f0e1da1520 +Author: Jim Ferenczi +Date: Tue Apr 16 09:29:22 2019 +0200 + + Upgrade to lucene 8.1.0-snapshot-e460356abe (#40952) + +commit 0777223bab04710bcdc01bd80905b50bdf5900e7 +Author: Marios Trivyzas +Date: Thu May 23 10:10:07 2019 +0200 + + Allow `fields` to be set to `*` (#42301) + + Allow for SimpleQueryString, QueryString and MultiMatchQuery + to set the `fields` parameter to the wildcard `*`. If so, set + the leniency to `true`, to achieve the same behaviour as from the + `"default_field" : "*" setting. + + Furthermore, check if `*` is in the list of the `default_field` but + not necessarily as the 1st element. + + Closes: #39577 + (cherry picked from commit e75ff0c748e6b68232c2b08e19ac4a4934918264) + +commit fa98cbe32072946672bc1df26bc61231a7c72f2c +Author: Mengwei Ding +Date: Wed May 22 20:25:45 2019 -0700 + + Add .code_internal-* index pattern to kibana user (#42247) (#42387) + +commit 824ccfabd9a2f358ac312367ded323a62aaecffd +Author: James Baiera +Date: Wed May 22 16:21:45 2019 -0400 + + Backport 7.x - Add step to forcemerge enrich index after reindex (#41969) + + Adds a step in the policy execution that forcemerge's a new enrich index after reindex completes. + +commit a71d19e92a7ebca92a96920fb6f0ae8651a2ce91 +Author: Yannick Welsch +Date: Wed May 22 19:07:56 2019 +0200 + + Ensure testAckedIndexing uses disruption index settings + + AbstractDisruptionTestCase set a lower global checkpoint sync interval setting, but this was ignored by + testAckedIndexing, which has led to spurious test failures + + Relates #41068, #38931 + +commit 496fee33338c4805dabe84f024ecbc7936bb884d +Author: Jake Landis +Date: Wed May 22 11:57:07 2019 -0500 + + bump to 7.3 (#42365) + +commit 4181c5ccf512e9c9f9fbe5f4584d07717739b4a5 +Author: swstepp <49322243+swstepp@users.noreply.github.com> +Date: Wed May 22 10:44:41 2019 -0600 + + Fix grammar problem in stemming reference. (#42148) + +commit a3caed2bee62cbb5e1f2ccbc55cb17f023b9982a +Author: Julie Tibshirani +Date: Wed May 22 09:19:14 2019 -0700 + + Fix a rendering issue in the geo envelope docs. (#42332) + + Previously the formatting information didn't display in the docs, and the + sentence just rendered as "bounding rectangle in the format :". + +commit c2af62455f6a001fc600bdf9c750b81b21adda34 +Author: Luca Cavanna +Date: Thu May 9 11:48:05 2019 +0200 + + Cut over SearchResponse and SearchTemplateResponse to Writeable (#41855) + + Relates to #34389 + +commit 29c9bb9181e5a00816f6aa0d41700c4442775b6c +Author: Luca Cavanna +Date: Thu May 9 11:47:36 2019 +0200 + + Clean up ShardId usage of Streamable (#41843) + + ShardId already implements Writeable so there is no need for it to implement Streamable too. Also the readShardId static method can be + easily replaced with direct usages of the constructor that takes a + StreamInput as argument. + +commit 96ba0b13e0a637318bfa36bad7e1a3db0fb2c03f +Author: Luca Cavanna +Date: Wed May 8 14:20:36 2019 +0200 + + Cut over MultiSearchResponse to Writeable (#41844) + + Relates to #34389 + +commit 1ded45b0a27e27d44a198cc6da5c8b40e039db98 +Author: Luca Cavanna +Date: Wed May 8 11:00:59 2019 +0200 + + Cut over SearchPhaseResult to Writeable (#41853) + + Relates to #34389 + +commit c85f285298eaf307c93ed3c912974cbc7e0af4b8 +Author: Luca Cavanna +Date: Tue May 7 10:54:47 2019 +0200 + + Move InternalAggregations to Writeable (#41841) + + Relates to #34389 + +commit 39d4c7c26f7f6acbe9423de4889e23a148eaee69 +Author: Luca Cavanna +Date: Mon May 6 10:58:31 2019 +0200 + + Skip explain fetch sub phase when request holds only suggestions (#41739) + + In case a search request holds only the suggest section, the query phase + is skipped and only the suggest phase is executed instead. There will + never be hits returned, and in case the explain flag is set to true, the + explain sub phase throws a null pointer exception as the query is null. + Usually a null query is replaced with a match all query as part of SearchContext#preProcess which is though skipped as well with suggest + only searches. To address this, we skip the explain sub fetch phase + for search requests that only requested suggestions. + + Closes #31260 + +commit 3416cda8b1aeac1d24714ddc674df63cc2a73559 +Author: Luca Cavanna +Date: Fri May 3 16:51:14 2019 +0200 + + Cut over ClusterSearchShardsGroup to Writeable (#41788) + +commit e747326b0482667b4d10867adca694fc5b67202d +Author: Luca Cavanna +Date: Fri May 3 11:15:24 2019 +0200 + + Adapt low-level REST client to java 8 (#41537) + + As a follow-up to #38540 we can use lambda functions and method + references where convenient in the low-level REST client. + + Also, we need to update the docs to state that the minimum java version + required is 1.8. + +commit 3e231bbad63fe6a4b0ace3bb3a43dee135ba4b83 +Author: Guillaume Darmont +Date: Sat May 4 17:00:34 2019 +0200 + + StackOverflowError when calling BulkRequest#add (#41672) + + Removing of payload in BulkRequest (#39843) had a side effect of making + `BulkRequest.add(DocWriteRequest...)` (with varargs) recursive, thus + leading to StackOverflowError. This PR adds a small change in + RequestConvertersTests to show the error and the corresponding fix in + `BulkRequest`. + + Fixes #41668 + +commit d4b5933225273d1c0b561b559c47c2612489a4a1 +Author: mushao999 +Date: Thu May 23 00:05:48 2019 +0800 + + Fix alpha version error message (#40406) + +commit 5d8605c7904521686a3da04da966f0fcd23c9409 +Author: Yannick Welsch +Date: Wed May 22 17:31:59 2019 +0200 + + Fix testAutoFollowManyIndices + + On a slow CI worker, the test was failing an assertion. + + Closes #41234 + +commit 9e514cb16120128b4be7e1213817af87052445f9 +Author: Martijn van Groningen +Date: Wed May 22 17:07:19 2019 +0200 + + Remove schedule field from EnrichPolicy (#42143) + +commit 075cc7c5cf708c977a88634a2fd80ec7a12f9c7c +Author: David Kyle +Date: Wed May 22 15:35:08 2019 +0100 + + [ML Data Frame] Persist data frame after state changes (#42347) + +commit f696769a392a54fbd34e71412133da4ee4fb367b +Author: David Kyle +Date: Wed May 22 14:56:14 2019 +0100 + + Mute Data Frame integration tests + + Relates to https://github.com/elastic/elasticsearch/issues/42344 + +commit eae58c477c8b21ad88496deed6d6022e9ca32122 +Author: Yannick Welsch +Date: Wed May 22 14:45:26 2019 +0200 + + Remove testNodeFailuresAreProcessedOnce + + This test was not checking the thing it was supposed to anyway. + +commit 250973af1dce7cd96433895ae98dadd8f4eaa89f +Author: Yannick Welsch +Date: Wed May 22 14:36:17 2019 +0200 + + Fix testCannotJoinIfMasterLostDataFolder + + Relates to #41047 + +commit a79cd77e5c9fd58bf58c6d9334b8db4c004b1759 +Author: Simon Willnauer +Date: Wed May 22 11:54:28 2019 +0200 + + Remove IndexShard dependency from Repository (#42213) + + * Remove IndexShard dependency from Repository + + In order to simplify repository testing especially for BlobStoreRepository + it's important to remove the dependency on IndexShard and reduce it to + Store and MapperService (in the snapshot case). This significantly reduces + the dependcy footprint for Repository and allows unittesting without starting + nodes or instantiate entire shard instances. This change deprecates the old + method signatures and adds a unittest for FileRepository to show the advantage + of this change. + In addition, the unittesting surfaced a bug where the internal file names that + are private to the repository were used in the recovery stats instead of the + target file names which makes it impossible to relate to the actual lucene files + in the recovery stats. + + * don't delegate deprecated methods + + * apply comments + + * test + +commit 3a20ff7e865c6a704987fe1a517ed5dc3dd67bdb +Author: Ignacio Vera +Date: Wed May 22 14:02:52 2019 +0200 + + Fix TopHitsAggregationBuilder adding duplicate _score sort clauses (#42179) (#42343) + + When using High Level Rest Client Java API to produce search query, using AggregationBuilders.topHits("th").sort("_score", SortOrder.DESC) + caused query to contain duplicate sort clauses. + +commit eb1639c5fc7a20b566b55605b33aa988196f982f +Author: Alpar Torok +Date: Wed May 22 14:25:54 2019 +0300 + + TestClusters: Convert docs (#42100) + + * TestClusters: Convert docs + +commit f338005179c67c0b6296fe4d6f612c0fc7afad3a +Author: Yannick Welsch +Date: Tue May 7 15:07:52 2019 +0200 + + Revert "Mute MinimumMasterNodesIT.testThreeNodesNoMasterBlock()" + + This reverts commit 448fc8444559be3145e4a7f65dec794ebbff7b81. + +commit 0c7322ebf2048a688c8af20908ae5e57c388e6ba +Author: Yannick Welsch +Date: Wed May 22 12:25:48 2019 +0200 + + Avoid bubbling up failures from a shard that is recovering (#42287) + + A shard that is undergoing peer recovery is subject to logging warnings of the form + + org.elasticsearch.action.FailedNodeException: Failed node [XYZ] + ... + Caused by: org.apache.lucene.index.IndexNotFoundException: no segments* file found in ... + + These failures are actually harmless, and expected to happen while a peer recovery is ongoing (i.e. + there is an IndexShard instance, but no proper IndexCommit just yet). + As these failures are currently bubbled up to the master, they cause unnecessary reroutes and + confusion amongst users due to being logged as warnings. + + Closes #40107 + +commit aab97f13118234a0d660b2020882078356063b76 +Author: Ioannis Kakavas +Date: Wed May 22 13:20:18 2019 +0300 + + Fail early when rp.client_secret is missing in OIDC realm (#42256) + + rp.client_secret is a required secure setting. Make sure we fail with + a SettingsException and a clear, actionable message when building + the realm, if the setting is missing. + +commit ccdc0e6b3e8620050d42693ce90e972c92bd6759 +Author: Ioannis Kakavas +Date: Wed May 22 12:00:21 2019 +0300 + + Merge claims from userinfo and ID Token correctly (#42277) + + Enhance the handling of merging the claims sets of the + ID Token and the UserInfo response. JsonObject#merge would throw a + runtime exception when attempting to merge two objects with the + same key and different values. This could happen for an OP that + returns different vales for the same claim in the ID Token and the + UserInfo response ( Google does that for profile claim ). + If a claim is contained in both sets, we attempt to merge the + values if they are objects or arrays, otherwise the ID Token claim + value takes presedence and overwrites the userinfo response. + +commit 7af30345b4e0aeac42a29fd6f35f46e52a1124a2 +Author: Ioannis Kakavas +Date: Wed May 22 11:10:23 2019 +0300 + + Revert "mute failing filerealm hash caching tests (#42304)" + + This reverts commit 39fbed1577ba4c0f118e30953e80a0969058a5be. + +commit a6eb20ad359cf4cc27d1b1c831229155df30ed57 +Author: Dimitris Athanasiou +Date: Wed May 22 12:42:04 2019 +0300 + + [ML] Include node name when native controller cannot start process (#42225) (#42338) + + This adds the node name where we fail to start a process via the native + controller to facilitate debugging as otherwise it might not be known + to which node the job was allocated. + +commit 770d8e9e39c78bc1e7d7521a7782194eaab36624 +Author: Yannick Welsch +Date: Wed May 22 10:25:11 2019 +0200 + + Remove usage of max_local_storage_nodes in test infrastructure (#41652) + + Moves the test infrastructure away from using node.max_local_storage_nodes, allowing us in a + follow-up PR to deprecate this setting in 7.x and to remove it in 8.0. + + This also changes the behavior of InternalTestCluster so that starting up nodes will not automatically + reuse data folders of previously stopped nodes. If this behavior is desired, it needs to be explicitly + done by passing the data path from the stopped node to the new node that is started. + +commit b1c413ea63dc9a216f1be9427b3256664b9e804d +Author: David Turner +Date: Wed May 22 09:26:41 2019 +0100 + + Rework discovery-ec2 docs (#41630) + + This commit reworks and clarifies the docs for the `discovery-ec2` plugin: + + - folds the tiny "Getting started with AWS" into the page on configuration + - spells out the name of each setting in full instead of noting the + `discovery.ec2` prefix at the top of the page. + - replaces each `(Secure)` marker with a sentence describing what that means in + situ + - notes some missing defaults + - clarifies the behaviour of `discovery.ec2.groups` (dependent on `.any_group`) + - clarifies what `discovery.ec2.host_type` is for + - adds `discovery.ec2.tag.TAGNAME` as a (meta-)setting rather than describing + it in a separate section + - notes that the tags mentioned in `discovery.ec2.tag.TAGNAME` cannot contain + colons (see #38406) + - clarifies the EC2-specific interface names and what they're for + - reorders and rewords the recommendations for storage + - expands on why you should not span a cluster across regions + - adds a suggestion on protecting instances against termination during scale-in + - reformat to 80 columns where possible + + Fixes #38406 + +commit c9dedf180b073a4925681a53d0735c1e17431a99 +Author: Yannick Welsch +Date: Wed May 22 09:35:07 2019 +0200 + + Use comparator for Reconfigurator (#42283) + + Simplifies the voting configuration reconfiguration logic by switching to an explicit Comparator for + the priorities. Does not make changes to the behavior of the component. + +commit ad24231c1a95825432bdbe2f88a4e1c5ab12f30b +Author: Hendrik Muhs +Date: Wed May 22 08:13:32 2019 +0200 + + [ML-DataFrame] validate group name to not contain invalid characters (#42292) + + disallows of creating groupBy field with '[', ']', '>' in the name to be consistent with aggregations + +commit 3493f3b637ee520a3ac4a9bd4b57adeffa515c0b +Author: Hendrik Muhs +Date: Wed May 22 08:12:58 2019 +0200 + + move latch await to doNextSearch (#42275) + + move latch await to doNextSearch, fixes a race condition when the executor thread is faster than the coordinator thread + + fixes #42084 + +commit 34dda75cdf8954c06598147ee3aedc0464abcd18 +Author: Ioannis Kakavas +Date: Wed May 22 08:20:51 2019 +0300 + + Ensure SHA256 is not used in tests (#42289) + + SHA256 was recently added to the Hasher class in order to be used + in the TokenService. A few tests were still using values() to get + the available algorithms from the Enum and it could happen that + SHA256 would be picked up by these. + This change adds an extra convenience method + (Hasher#getAvailableAlgoCacheHash) and enures that only this and + Hasher#getAvailableAlgoStoredHash are used for getting the list of + available password hashing algorithms in our tests. + +commit cdf9485e331bfa31253bb4e1e6b4e5d537bd0abf +Author: Ioannis Kakavas +Date: Wed May 22 08:20:25 2019 +0300 + + Allow Kibana user to use the OpenID Connect APIs (#42305) + + Add the manage_oidc privilege to the kibana user and to the role + privileges list + +commit c5f191f6afe59a4c805c4a6d0b7b22d566f7b73c +Author: Tim Vernum +Date: Wed May 22 14:27:45 2019 +1000 + + Add cluster restart for security on basic (#42217) + + This performs a simple restart test to move a basic licensed + cluster from no security (the default) to security & transport TLS + enabled. + + Backport of: #41933 + +commit bcbf1aff6b03ff915c301c7d05166adbc667b0c1 +Author: Nhat Nguyen +Date: Tue May 21 22:35:51 2019 -0400 + + Peer recovery should flush at the end (#41660) + + Flushing at the end of a peer recovery (if needed) can bring these + benefits: + + 1. Closing an index won't end up with the red state for a recovering + replica should always be ready for closing whether it performs the + verifying-before-close step or not. + + 2. Good opportunities to compact store (i.e., flushing and merging + Lucene, and trimming translog) + + Closes #40024 + Closes #39588 + +commit 84df48ccb3b07e8f2a56b95256537ae7b4c29fef +Author: Nhat Nguyen +Date: Tue May 21 17:20:27 2019 -0400 + + Recovery with syncId should verify seqno infos (#41265) + + This change verifies and aborts recovery if source and target have the + same syncId but different sequenceId. This commit also adds an upgrade + test to ensure that we always utilize syncId. + +commit 3573b1d0ce6915f9a40569d60da5df71715e281a +Author: Nhat Nguyen +Date: Tue May 21 19:54:46 2019 -0400 + + Skip global checkpoint sync for closed indices (#41874) + + The verifying-before-close step ensures the global checkpoints on all + shard copies are in sync; thus, we don' t need to sync global + checkpoints for closed indices. + + Relate #33888 + +commit 4d55e9e070344c0d8c8318436bcbaefce4d5f5d1 +Author: Nhat Nguyen +Date: Tue May 21 19:45:52 2019 -0400 + + Estimate num history ops should always use translog (#42211) + + Currently, we ignore soft-deletes in peer recovery, thus + estimateNumberOfHistoryOperations should always use translog. + + Relates #38904 + +commit b510402b6762e02a80467a5a3c609c600fa9a91a +Author: Jason Tedor +Date: Tue May 21 19:13:39 2019 -0400 + + Fix off-by-one error in an index shard test + + There is an off-by-one error in this test. It leads to the recovery + thread never being started, and that means joining on it will wait + indefinitely. This commit addresses that by fixing the off-by-one error. + + Relates #42325 + +commit 323251c3d1cf8a3e33b7565d6b1c4337ebf54083 +Merge: 855f5cc6a5c ecd033bea62 +Author: Michael Basnight +Date: Tue May 21 16:51:42 2019 -0500 + + Merge branch '7.x' into enrich-7.x + +commit 6808951e6f5e465b9b2ae2a434229bc9636519fa +Author: Nhat Nguyen +Date: Tue May 21 17:06:51 2019 -0400 + + Mute testDelayedOperationsBeforeAndAfterRelocated + + Tracked at #42325 + +commit dd7a65fdf2a0ddc8cea9ef6cbf6579d0f7a4a5f1 +Author: Jason Tedor +Date: Tue May 21 16:12:27 2019 -0400 + + Fix compilation in IndexShardTests + + I forgot to git add these before pushing, sorry. This commit fixes + compilation in IndexShardTests, they are needed here and not in master + due to differences in how Java infers types in generics between JDK 8 + and JDK 11. + +commit f7ff0aff798784b4e0d2203d5df7c60ce68809fb +Author: Jason Tedor +Date: Tue May 21 15:53:28 2019 -0400 + + Execute actions under permit in primary mode only (#42241) + + Today when executing an action on a primary shard under permit, we do + not enforce that the shard is in primary mode before executing the + action. This commit addresses this by wrapping actions to be executed + under permit in a check that the shard is in primary mode before + executing the action. + +commit 685a206891e9741ec80d1e5951a3cc0f4887ca1b +Merge: d97f4d5e286 32b70ed34cb +Author: Ed Savage +Date: Tue May 21 19:14:17 2019 +0100 + + Merge branch '7.x' of github.com:elastic/elasticsearch into 7.x + +commit 32b70ed34cbafb9979e7280a34938b1f675d8311 +Author: Jason Tedor +Date: Tue May 21 13:45:58 2019 -0400 + + Avoid unnecessary persistence of retention leases (#42299) + + Today we are persisting the retention leases at least every thirty + seconds by a scheduled background sync. This sync causes an fsync to + disk and when there are a large number of shards allocated to slow + disks, these fsyncs can pile up and can severely impact the system. This + commit addresses this by only persisting and fsyncing the retention + leases if they have changed since the last time that we persisted and + fsynced the retention leases. + +commit 7e4d3c695b4c81cf10db166d6fc48261cb2fec0a +Author: David Kyle +Date: Tue May 21 18:40:04 2019 +0100 + + [ML Data Frame] Persist and restore checkpoint and position (#41942) + + Persist and restore Data frame's current checkpoint and position + +commit 813db163d8fc041b40d1a71957695b6b27a14acb +Author: Jack Conradson +Date: Tue May 21 13:47:47 2019 -0400 + + Reorganize Painless doc structure (#42303) + +commit d97f4d5e2867292d2e3676daa11615c003457a5f +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Tue May 21 18:41:59 2019 +0100 + + [ML][TEST] Fix limits in AutodetectMemoryLimitIT (#42279) + + Re-enable muted tests and accommodate recent backend changes + that result in higher memory usage being reported for a job + at the start of its life-cycle + +commit 39fbed1577ba4c0f118e30953e80a0969058a5be +Author: Tal Levy +Date: Tue May 21 10:38:40 2019 -0700 + + mute failing filerealm hash caching tests (#42304) + + some tests are failing after the introduction of #41792. + + relates #42267 and #42289. + +commit ecd033bea62489e70c60763cfc117d0b96534fa0 +Author: Armin Braun +Date: Tue May 21 17:20:52 2019 +0200 + + Cleanup Various Uses of ActionListener (#40126) (#42274) + + * Cleanup Various Uses of ActionListener + + * Use shorter `map`, `runAfter` or `wrap` where functionally equivalent to anonymous class + * Use ActionRunnable where functionally equivalent + +commit a4e6fb4dd2c2ec24638f675a5901b28809c4e6e7 +Author: Dimitris Athanasiou +Date: Tue May 21 18:03:24 2019 +0300 + + [ML] Fix logger declaration in ML plugins (#42222) (#42238) + + This corrects what appears to have been a copy-paste error + where the logger for `MachineLearning` and `DataFrame` was wrongly + set to be that of `XPackPlugin`. + +commit 0fd42ce1f5944b094fbddfdef05a27cd3e6c1b60 +Author: David Kyle +Date: Tue May 21 15:48:45 2019 +0100 + + [ML Data Frame] Start directly data frame rather than via the scheduler (#42224) + + Trigger indexer start directly to put the indexer in INDEXING state immediately + +commit 75425ae167d1f2c2ac489f0f389e62d2d37de4a5 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue May 21 15:51:55 2019 +0200 + + Remove 7.0.2 (#42282) + + 7.0.2 removed, since it will never be, fixing branch consistency check. + +commit cef4b9ba76ff297efbebbf48483b0ad7b6e07a02 +Author: Alpar Torok +Date: Tue May 21 16:37:11 2019 +0300 + + Move the FIPS configuration back to the build plugin (#41989) + + * Move the FIPS configuration back to the build plugin + + This is necesary for external users of build-tools. + Closes #41721 + +commit a6204a5eafa6460d9043a9b0b6f934048527138c +Author: Glen Smith +Date: Tue May 21 14:59:40 2019 +0200 + + Remove stray back tick that's messing up table format (#41705) + +commit de096485c8c1878aaf22405d6d619216402a584a +Author: Alpar Torok +Date: Tue May 21 13:49:42 2019 +0300 + + Use spearate testkit dir for each run (#42013) + + Gradle Testkit reuses the teskit dir by default between tests. + With this change we use a temporary one for each run + hoping it will fix #41431 + +commit 216c74d10aebc0566b8de15088ff660f33af38d5 +Author: Mayya Sharipova +Date: Tue May 21 06:36:38 2019 -0400 + + Add experimental and warnings to vector functions (#42205) + +commit 0449869511dc75c399a3e60a581ae3d93cf77b74 +Author: jimczi +Date: Tue May 21 12:28:57 2019 +0200 + + Fix unchecked warning in RollupIndexerIndexingTests#testSimpleDateHistoWithOverlappingDelay + +commit ffefc66260ae3791f4fb561ee27e91cf964ea8a6 +Author: David Kyle +Date: Fri May 10 16:28:28 2019 +0100 + + Mute failing AsyncTwoPhaseIndexerTests + + See https://github.com/elastic/elasticsearch/issues/42084 + +commit 24144aead2d4598711af51c979ed238933548201 +Author: David Kyle +Date: Tue May 21 10:24:20 2019 +0100 + + [ML] Complete the Data Frame task on stop (#41752) (#42063) + + Wait for indexer to stop then complete the persistent task on stop. + If the wait_for_completion is true the request will not return until stopped. + +commit 7b3a9c7033f9c4a348938f4c8caeb15cfe704c46 +Author: Tim Vernum +Date: Tue May 21 18:14:22 2019 +1000 + + Do not refresh realm cache unless required (#42212) + + If there are no realms that depend on the native role mapping store, + then changes should it should not perform any cache refresh. + A refresh with an empty realm array will refresh all realms. + + This also fixes a spurious log warning that could occur if the + role mapping store was notified that the security index was recovered + before any realm were attached. + + Backport of: #42169 + +commit 7abeaba8bb374130283978b7505c1943ef580ad6 +Author: David Turner +Date: Tue May 21 07:52:01 2019 +0100 + + Prevent in-place downgrades and invalid upgrades (#41731) + + Downgrading an Elasticsearch node to an earlier version is unsupported, because + we do not make any attempt to guarantee that a node can read any of the on-disk + data written by a future version. Yet today we do not actively prevent + downgrades, and sometimes users will attempt to roll back a failed upgrade with + an in-place downgrade and get into an unrecoverable state. + + This change adds the current version of the node to the node metadata file, and + checks the version found in this file against the current version at startup. + If the node cannot be sure of its ability to read the on-disk data then it + refuses to start, preserving any on-disk data in its upgraded state. + + This change also adds a command-line tool to overwrite the node metadata file + without performing any version checks, to unsafely bypass these checks and + recover the historical and lenient behaviour. + +commit ec63160243e1b4a888d567fe4efd6dff9717a7c1 +Author: Jim Ferenczi +Date: Mon May 20 09:55:35 2019 -0400 + + Fix max boundary for rollups job that use a delay (#42158) + + Rollup jobs can define how long they should wait before rolling up new documents. + However if the delay is smaller or if it's not a multiple of the rollup interval + the job can create incomplete buckets because the max boundary for a job is computed + from the time when the job started rounded to the interval minus the delay. This change + fixes this computation by applying the delay substraction before the rounding in order to ensure + that we never create a boundary that falls in a middle of a bucket. + +commit b0a25c3170d61e1cc2b2e901caf102b83b88845a +Author: Jake Landis +Date: Mon May 20 17:58:24 2019 -0500 + + add 7.1.1 and 6.8.1 versions (#42251) + +commit df8fef3c1ad2eb1ebc6aef3f458f6c5454ccdc68 +Author: Jake Landis +Date: Mon May 20 15:32:44 2019 -0400 + + fix assumption that 6.7 is last 6.x release (#42255) + +commit 87bff89500b93fe46fcf088c7e4238c06cac73cb +Author: Jake Landis +Date: Mon May 20 14:39:17 2019 -0400 + + 7.1.0 release notes forward port (#42252) + + Forward port of #42208 + +commit be515d7ce0db7d9143bbe8dbc799332d39c64009 +Author: Ryan Ernst +Date: Mon May 20 11:35:31 2019 -0700 + + Validate non-secure settings are not in keystore (#42209) + + Secure settings currently error if they exist inside elasticsearch.yml. + This commit adds validation that non-secure settings do not exist inside + the keystore. + + closes #41831 + +commit 6ae6f57d39f473e4968700a28a582b93fe3a3bf4 +Author: Zachary Tong +Date: Mon May 20 12:07:29 2019 -0400 + + [7.x Backport] Force selection of calendar or fixed intervals (#41906) + + The date_histogram accepts an interval which can be either a calendar + interval (DST-aware, leap seconds, arbitrary length of months, etc) or + fixed interval (strict multiples of SI units). Unfortunately this is inferred + by first trying to parse as a calendar interval, then falling back to fixed + if that fails. + + This leads to confusing arrangement where `1d` == calendar, but + `2d` == fixed. And if you want a day of fixed time, you have to + specify `24h` (e.g. the next smallest unit). This arrangement is very + error-prone for users. + + This PR adds `calendar_interval` and `fixed_interval` parameters to any + code that uses intervals (date_histogram, rollup, composite, datafeed, etc). + Calendar only accepts calendar intervals, fixed accepts any combination of + units (meaning `1d` can be used to specify `24h` in fixed time), and both + are mutually exclusive. + + The old interval behavior is deprecated and will throw a deprecation warning. + It is also mutually exclusive with the two new parameters. In the future the + old dual-purpose interval will be removed. + + The change applies to both REST and java clients. + +commit c72c76b5ea0dbdfe220de4ad0e623772b8de582f +Author: Alexander Reelsen +Date: Mon May 20 10:17:08 2019 -0400 + + Update to joda time 2.10.2 (#42199) + +commit b4a413c4d0c03f5beca7757c8544f84e61fd85d8 +Author: Ioannis Kakavas +Date: Mon May 20 17:55:29 2019 +0300 + + Hash token values for storage (#41792) (#42220) + + This commit changes how access tokens and refresh tokens are stored + in the tokens index. + + Access token values are now hashed before being stored in the id + field of the `user_token` and before becoming part of the token + document id. Refresh token values are hashed before being stored + in the token field of the `refresh_token`. The tokens are hashed + without a salt value since these are v4 UUID values that have + enough entropy themselves. Both rainbow table attacks and offline + brute force attacks are impractical. + + As a side effect of this change and in order to support multiple + concurrent refreshes as introduced in #39631, upon refreshing an + pair, the superseding access token + and refresh tokens values are stored in the superseded token doc, + encrypted with a key that is derived from the superseded refresh + token. As such, subsequent requests to refresh the same token in + the predefined time window will return the same superseding access + token and refresh token values, without hitting the tokens index + (as this only stores hashes of the token values). AES in GCM + mode is used for encrypting the token values and the key + derivation from the superseded refresh token uses a small number + of iterations as it needs to be quick. + + For backwards compatibility reasons, the new behavior is only + enabled when all nodes in a cluster are in the required version + so that old nodes can cope with the token values in a mixed + cluster during a rolling upgrade. + +commit 072a9bdf55b1114c59bdd026025f5f71e6665f7f +Author: Zachary Tong +Date: Mon May 20 09:46:37 2019 -0400 + + Fix FiltersAggregation NPE when `filters` is empty (#41459) + + If `keyedFilters` is null it assumes there are unkeyed filters...which + will NPE if the unkeyed filters was actually empty. + + This refactors to simplify the filter assignment a bit, adds an empty + check and tidies up some formatting. + +commit dbbdcea128af746d94d4b5a246026929d2b5bca2 +Author: Jay Modi +Date: Mon May 20 09:45:36 2019 -0400 + + Update ciphers for TLSv1.3 and JDK11 if available (#42082) + + This commit updates the default ciphers and TLS protocols that are used + when the runtime JDK supports them. New cipher support has been + introduced in JDK 11 and 12 along with performance fixes for AES GCM. + The ciphers are ordered with PFS ciphers being most preferred, then + AEAD ciphers, and finally those with mainstream hardware support. When + available stronger encryption is preferred for a given cipher. + + This is a backport of #41385 and #41808. There are known JDK bugs with + TLSv1.3 that have been fixed in various versions. These are: + + 1. The JDK's bundled HttpsServer will endless loop under JDK11 and JDK + 12.0 (Fixed in 12.0.1) based on the way the Apache HttpClient performs + a close (half close). + 2. In all versions of JDK 11 and 12, the HttpsServer will endless loop + when certificates are not trusted or another handshake error occurs. An + email has been sent to the openjdk security-dev list and #38646 is open + to track this. + 3. In JDK 11.0.2 and prior there is a race condition with session + resumption that leads to handshake errors when multiple concurrent + handshakes are going on between the same client and server. This bug + does not appear when client authentication is in use. This is + JDK-8213202, which was fixed in 11.0.3 and 12.0. + 4. In JDK 11.0.2 and prior there is a bug where resumed TLS sessions do + not retain peer certificate information. This is JDK-8212885. + + The way these issues are addressed is that the current java version is + checked and used to determine the supported protocols for tests that + provoke these issues. + +commit fd2d4d761b5724b1f64e1b5be71d47ade04de6b1 +Author: Lisa Cawley +Date: Mon May 20 09:06:42 2019 -0400 + + [DOCS] Updates TLS configuration info (#41983) + +commit 4e9bf3f18a540be97c4261f5176d93ba02f96a08 +Author: Tomas Della Vedova +Date: Mon May 20 06:40:45 2019 -0400 + + Remove deprecated _source_exclude and _source_include from get API spec (#42188) + + Support for these parameters was removed in #35097. The spec were left outdated. + +commit 855f5cc6a5c3615dbc4f3fc711c538d823aeb19a +Merge: 2daf568565e 8f838198fa8 +Author: Martijn van Groningen +Date: Mon May 20 12:16:57 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 8f838198fa89841d177a07a9fe65ab89cbed1b0e +Author: Russ Cam +Date: Mon May 20 06:07:28 2019 -0400 + + Remove parent query string parameter (#41098) + + This commit removes the deprecated parent query string + parameter. The routing parameter should be used instead. + +commit b7599472aced11c47c510823ffadfb03050abe32 +Author: Jim Ferenczi +Date: Mon May 20 04:02:01 2019 -0400 + + Fix random failure in SearchRequestTests#testRandomVersionSerialization (#42069) + + This commit fixes a test bug that ends up comparing the result of two consecutive calls to System.currentTimeMillis that can be different + on slow CIs. + + Closes #42064 + +commit 0ec7986049ee9e65cc5c630ae6ff1401cc08ae0c +Author: Nhat Nguyen +Date: Sun May 19 22:05:52 2019 -0400 + + Enable debug log in testRetentionLeasesSyncOnRecovery + + Relates #39105 + +commit 1362944c23694fe7df8da50d36f95c1b0144e404 +Author: Nhat Nguyen +Date: Sun May 19 20:43:41 2019 -0400 + + Minor improvement translog docs (#42184) + + Closes #42183 + +commit 840af87a7496b64c35c643f3682d6a3aa2f629ac +Author: Ed Savage +Date: Sun May 19 08:29:50 2019 -0400 + + [ML] Temporarily muting failing tests + + Muting a number of AutoDetectMemoryLimitIT tests to give CI a chance to + settle before easing in required backend changes. + + relates elastic/ml-cpp#486 + relates #42086 + +commit a68b04e47bf91aa7f301643017eaa5a2d145216e +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Fri May 17 16:37:52 2019 -0400 + + [ML] Improve hard_limit audit message (#42086) + + Improve the hard_limit memory audit message by reporting how many bytes + over the configured memory limit the job was at the point of the last + allocation failure. + + Previously the model memory usage was reported, however this was + inaccurate and hence of limited use - primarily because the total + memory used by the model can decrease significantly after the models + status is changed to hard_limit but before the model size stats are + reported from autodetect to ES. + + While this PR contains the changes to the format of the hard_limit audit + message it is dependent on modifications to the ml-cpp backend to + send additional data fields in the model size stats message. These + changes will follow in a subsequent PR. It is worth noting that this PR + must be merged prior to the ml-cpp one, to keep CI tests happy. + +commit f2447364fd34228d19551e1989f2f7155caae184 +Author: Benjamin Trent +Date: Fri May 17 16:51:05 2019 -0400 + + [ML] adds geo_centroid aggregation support to data frames (#42088) (#42094) + +commit 076ca75ea52309d4be43fdf407e994fb5086f623 +Author: Igor Motov +Date: Fri May 17 15:59:34 2019 -0400 + + SQL: Suppress geo tests failing on tr-TR locale (#42200) + + Due to a bug in JTS WKT parser, JTS cannot parse most of WKT shapes if + the shape type is written in the lower case. For examples `point (1 2)` + is causing JTS inside H2GIS to fail on tr-TR locale as a result of + case-insensitive comparison. + +commit ab7a7062ea1401c3e02223f5677e5d7b63137c54 +Author: Ryan Ernst +Date: Fri May 17 14:49:29 2019 -0400 + + Make packaging tests use jdk downloader (#42097) + + This commit removes the jdk11 download in vagrant provisioning and + converts it to using the jdk downloader for the system jdk, and sets up + a separate jdk for use by the test (which will be converted to running + gradle in a followup). + +commit a6e63e6fa8868da4138b18df62e3f5a4768ed6d0 +Author: Ryan Ernst +Date: Thu May 16 15:38:15 2019 -0400 + + Protect logged exec spooling from no output (#42177) + + This commit adds a guard around reading the spooled LoggedExec output. + It is possible the exec command did not output anything, and failed, + which would trigger a failure to read the output file. + +commit c40bd31073a5c721bf7bd4d112cd7a46f377de2c +Author: Ryan Ernst +Date: Thu May 16 15:35:27 2019 -0400 + + Use local outputstream reference (#42180) + + This commit fixes the logging in LoggedExec which uses an in memory + buffer to read from a local reference, instead of with + getStandardOutput() of the Exec task. This is due to gradle internally + wrapping with a TeeOutputStream, breaking our cast. + +commit 51376f98a731c722786976cabb41706d02d11e9f +Author: David Turner +Date: Thu May 16 13:36:09 2019 -0400 + + Clarify rolling upgrade fallback to restart upgrade (#42161) + + Adds a note that restarting half-or-more of the master-eligible nodes means + you're no longer doing a rolling upgrade, and may need to upgrade all the + things before the cluster returns to health. + +commit 226df35d96f1a9357d25a1b124fd01cbbb822350 +Author: David Roberts +Date: Thu May 16 13:11:23 2019 -0400 + + [ML] Improve message misformation error in file structure finder (#42175) + + This change replaces the extremely unfriendly message + "Number of messages analyzed must be positive" in the + case where the sample lines were incorrectly grouped + into just one message to an error that more helpfully + explains the likely root cause of the problem. + +commit 4063701f5e52935488dad16ba085febfdf6e2fbf +Author: Hendrik Muhs +Date: Thu May 16 10:10:23 2019 -0400 + + [DOCS] add a warning about bypassing PUT API's, update example responses (#42062) + + Configurations are stored in the .data-frame-internal-1 + index, but users should not add configurations directly to + the index as additional information to enable access control + is added. This adds a warning against allowing access to the + internal index. + +commit fa1d1d1f577d4674b0214753b3c1481e1a2b3e4c +Author: Ryan Ernst +Date: Thu May 16 09:52:13 2019 -0400 + + Deprecate the native realm migration tool (#42142) + + The migrate tool was added when the native realm was created, to aid + users in converting from file realms that were per node, into the + cluster managed native realm. While this tool was useful at the time, + users should now be using the native realm directly. This commit + deprecates the tool, to be removed in a followup for 8.0. + +commit 8681dd9cba2260237480d6aea4479a4449a9e9e6 +Author: Ryan Ernst +Date: Thu May 16 09:48:57 2019 -0400 + + Hide bwc build output on success (#42102) + + Previously we used LoggedExec for running the internal bwc builds. + However, this had bad performance implications as all the output was + buffered into memory, thus we changed back to normal Exec. This commit + adds a `spoolOutput` setting to LoggedExec which can be used for + commands with large amounts of output, and switches the bwc builds to + use this flag. + +commit 6ffc6ea42e101050624a4a5f36e2bfd61c417c67 +Author: Nhat Nguyen +Date: Wed May 15 18:13:04 2019 -0400 + + Don't verify evictions in testFilterCacheStats (#42091) + + If a background merge and refresh happens after a search but before a + stats query, then evictions will be non-zero. + + Closes #32506 + +commit a75e916078d6083c76b7260287b60db3f022914f +Author: Nhat Nguyen +Date: Tue May 14 07:45:14 2019 -0400 + + Adjust load and timeout in testShrinkIndexPrimaryTerm (#42098) + + This test can create and shuffle 2*(3*5*7) = 210 shards which is quite + heavy for our CI. This commit reduces the load, so we don't timeout on + CI. + + Closes #28153 + +commit 7473742e6eb053f812d809576c78bdf80476e422 +Author: Marios Trivyzas +Date: Wed May 15 16:06:03 2019 -0400 + + SQL: Fix issue regarding INTERVAL * number (#42014) + + Interval * integer number is a valid operation which previously was + only supported for foldables (literals) and not when a field was + involved. That was because: + + 1. There was no common type returned for that combination + 2. The `BinaryArithmeticOperation` was permitting the multiplication + (called by fold()) but the BinaryArithmeticProcessor didn't allow it + + Moreover the error message for invalid arithmetic operations was wrong + because of the issue with the overloading methods of + `LoggerMessageFormat.format`. + + Fixes: #41239 + Fixes: #41200 + (cherry picked from commit 91039bab12d3ef27d6eac9cdc891a3b3ad0c694d) + +commit 9191b02213f2217ac97b199c82683088daa9f5be +Author: Tim Vernum +Date: Thu May 16 03:53:34 2019 +1000 + + Enforce transport TLS on Basic with Security (#42150) + + If a basic license enables security, then we should also enforce TLS + on the transport interface. + + This was already the case for Standard/Gold/Platinum licenses. + + For Basic, security defaults to disabled, so some of the process + around checking whether security is actuallY enabled is more complex + now that we need to account for basic licenses. + +commit 2f8c5ac6f878125a79ef0955047a019c6717d183 +Author: Igor Motov +Date: Wed May 15 10:50:54 2019 -0400 + + Docs: Mark SQL Geo functionality as beta (#42138) + + Adds beta marker to geosql documentation + +commit 15fd233ae36c11c7c48aaa7b76e2837e80124080 +Author: David Turner +Date: Wed May 15 09:26:04 2019 -0400 + + Minor cluster coordination docs fixes (#42111) + + Fixes a typo and a badly-formatted warning. + +commit a8aa818e0046a55d6a04f993726080dc45814880 +Author: Mark Vieira +Date: Wed May 15 08:11:55 2019 -0400 + + Cacheability improvements for thirdparty audit task (#42085) (#42151) + +commit 70ea3cf8474bae838fbad207ab5c9583b644c678 +Author: Igor Motov +Date: Tue May 14 18:57:12 2019 -0500 + + SQL: Add initial geo support (#42031) (#42135) + + Adds an initial limited implementations of geo features to SQL. This implementation is based on the [OpenGIS® Implementation Standard for Geographic information - Simple feature access](http://www.opengeospatial.org/standards/sfs), which is the current standard for GIS system implementation. This effort is concentrate on SQL option AKA ISO 19125-2. + + Queries that are supported as a result of this initial implementation + + Metadata commands + + - `DESCRIBE table` - returns the correct column types `GEOMETRY` for geo shapes and geo points. + - `SHOW FUNCTIONS` - returns a list that includes supported `ST_` functions + - `SYS TYPES` and `SYS COLUMNS` display correct types `GEO_SHAPE` and `GEO_POINT` for geo shapes and geo points accordingly. + + Returning geoshapes and geopoints from elasticsearch + + - `SELECT geom FROM table` - returns the geoshapes and geo_points as libs/geo objects in JDBC or as WKT strings in console. + - `SELECT ST_AsWKT(geom) FROM table;` and `SELECT ST_AsText(geom) FROM table;`- returns the geoshapes ang geopoints in their WKT representation; + + Using geopoints to elasticsearch + + - The following functions will be supported for geopoints in queries, sorting and aggregations: `ST_GeomFromText`, `ST_X`, `ST_Y`, `ST_Z`, `ST_GeometryType`, and `ST_Distance`. In most cases when used in queries, sorting and aggregations, these function are translated into script. These functions can be used in the SELECT clause for both geopoints and geoshapes. + - `SELECT * FROM table WHERE ST_Distance(ST_GeomFromText(POINT(1 2), point) < 10;` - returns all records for which `point` is located within 10m from the `POINT(1 2)`. In this case the WHERE clause is translated into a range query. + + Limitations: + + Geoshapes cannot be used in queries, sorting and aggregations as part of this initial effort. In order to fully take advantage of geoshapes we would need to have access to geoshape doc values, which is coming in #37206. `ST_Z` cannot be used on geopoints in queries, sorting and aggregations since we don't store altitude in geo_point doc values. + + Relates to #29872 + Backport of #42031 + +commit 327f44e05108558f74276748baa2a8d5a917db96 +Author: Jay Modi +Date: Tue May 14 16:29:18 2019 -0400 + + Concurrent tests wait for threads to be ready (#42083) + + This change updates tests that use a CountDownLatch to synchronize the + running of threads when testing concurrent operations so that we ensure + the thread has been fully created and run by the scheduler. Previously, + these tests used a latch with a value of 1 and the test thread counted + down while the threads performing concurrent operations just waited. + This change updates the value of the latch to be 1 + the number of + threads. Each thread counts down and then waits. This means that each + thread has been constructed and has started running. All threads will + have a common start point now. + +commit 367e0279623b250f70dd84fbcb69ab184998fae5 +Author: David Turner +Date: Tue May 14 10:34:20 2019 +0100 + + Log cluster UUID when committed (#42065) + + Today we do not expose the cluster UUID in any logs by default, but it would be + useful to see it. For instance if a user starts multiple nodes as separate + clusters then they will silently remain as separate clusters even if they are + subsequently reconfigured to look like a single cluster. This change logs the + committed cluster UUID the first time the node encounters it. + +commit 58f2e916848ecc4bec5b4f9e12fb30ca55507861 +Author: James Rodewig +Date: Mon May 13 08:42:26 2019 -0400 + + [DOCS] Rewrite 'rewrite' parameter docs (#42018) + +commit 2daf568565e751af273d745c63be0c6a987ca2b5 +Author: Martijn van Groningen +Date: Sun May 12 20:57:24 2019 +0200 + + Tidy up EnrichPolicy class (#41877) + +commit 90dce0864a169d24f6e61ac39df225a261c83a2d +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sun May 12 10:32:02 2019 +1000 + + Increase the sample space for random inner hits name generator (#42057) (#42072) + + This commits changes the minimum length for inner hits + name to avoid name collision which sometimes failed the + test. + +commit 912c6bdbfff805c5c93f22c4be41708cae76a031 +Author: Andrei Stefan +Date: Sat May 11 01:58:03 2019 +0300 + + Prevent order being lost for _nodes API filters (#42045) (#42089) + + * Switch to using a list instead of a Set for the filters, so that the + order of these filters is kept. + + (cherry picked from commit 74a743829799b64971e0ac5ae265f43f6c14e074) + +commit a85189a55864df9337eab1d9d58b44692424a5f9 +Author: Gordon Brown +Date: Fri May 10 16:06:42 2019 -0600 + + Remove toStepKeys from LifecycleAction (#41775) + + The `toStepKeys()` method was only called in its own test case. The real + list of StepKeys that's used in action execution is generated from the + list of actual step objects returned by `toSteps()`. + + This commit removes that method. + +commit c19ea0a6f18a3d2bf9c06f9d7e6efee1779686fb +Author: Nhat Nguyen +Date: Fri May 10 14:27:16 2019 -0400 + + Remove global checkpoint assertion in peer recovery (#41987) + + If remote recovery copies an index commit which has gaps in sequence + numbers to a follower; then these assertions (introduced in #40823) + don't hold for follower replicas. + + Closes #41037 + +commit febee07dccaf3a0776953bc8d028a8b2301b6254 +Author: Benjamin Trent +Date: Fri May 10 13:22:31 2019 -0500 + + [ML] adding pivot.max_search_page_size option for setting paging size (#41920) (#42079) + + * [ML] adding pivot.size option for setting paging size + + * Changing field name to address PR comments + + * fixing ctor usage + + * adjust hlrc for field name change + +commit 0931815355f017b25122a4a67d28a7146c25acc6 +Author: Benjamin Trent +Date: Fri May 10 13:22:12 2019 -0500 + + [ML] properly nesting objects in document source (#41901) (#42077) + + * [ML] properly nesting objects in document source + + * Throw exception on agg extraction failure, cause it to fail df + + * throwing error to stop df if unsupported agg is found + +commit 9944fdf237262d55b0b28a65934daa125f35338f +Author: Ryan Ernst +Date: Fri May 10 11:16:57 2019 -0700 + + Don't create tempdir for cli scripts (#41913) + + The elasticsearch-cli helper script does not use the tempdir created by + elasticsearch-env, yet the env script still creates it. This can lead to + lots of temp directories being created when running cli scripts in an + automated fashion. This commit passes a fake tmpdir to the env script to + avoid creation. + + closes #34445 + +commit 2244697219ee0a886c160fbbb021b076fbf5b647 +Author: Ryan Ernst +Date: Fri May 10 11:07:29 2019 -0700 + + Fix debian-8 update (#42056) + + On debian-8, when trying to apt-get update, it currently (sometimes) + fails on one of the extra repositories. This failure to update causes + keys to not be updated, which later can cause some packages to not + install due to lack of key verification. This commit removes the + troublesome repository before we attemp to update. + + closes #42017 + +commit 69824ed9084a06eba57f381d18610d3466e7c32f +Author: Ryan Ernst +Date: Fri May 10 11:00:17 2019 -0700 + + Cleanup plugin bin directories (#41907) + + This commit adds deletion of the bin directory to postrm cleanup. While + the package's bin files are cleaned up by the package manager, plugins + may have created subdirectories under bin. We already cleanup plugins, + but not the extra bin dirs their installation created. + + closes #18109 + +commit 3e59c31a127d387148e62879c14c681a80c5b9b6 +Author: Christoph Büscher +Date: Fri May 10 18:07:05 2019 +0200 + + Change IndexAnalyzers default analyzer access (#42011) + + Currently IndexAnalyzers keeps the three default as separate class members + although they should refer to the same analyzers held in the additional + analyzers map under the default names. This assumption should be made more + explicit by keeping all analyzers in the map. This change adapts the constructor + to check all the default entries are there and the getters to reach into the map + with the default names when needed. + +commit cd5f1b53e8f6052f6175f8f922fcfd40f518752a +Author: Jason Tedor +Date: Fri May 10 11:47:05 2019 -0400 + + Remove reference to fs.data.spins in docs + + We long ago removed fs.data.spins from the nodes stats. This commit + removes reference to this in the docs. + +commit 80432a35520c8cfe9d93ee45e7bf70a10be8ffd5 +Author: Jay Modi +Date: Fri May 10 08:43:35 2019 -0600 + + Remove close method in PageCacheRecycler/Recycler (#41917) + + The changes in #39317 brought to light some concurrency issues in the + close method of Recyclers as we do not wait for threads running in the + threadpool to be finished prior to the closing of the PageCacheRecycler + and the Recyclers that are used internally. #41695 was opened to + address the concurrent close issues but upon review, the closing of + these classes is not really needed as the instances should be become + available for garbage collection once there is no longer a reference to + the closed node. + + Closes #41683 + +commit 44c34185319e79c1831edd2653b6b3a46acf1d9c +Author: Alan Woodward +Date: Fri May 10 14:38:10 2019 +0100 + + Simplify handling of keyword field normalizers (#42002) + + We have a number of places in analysis-handling code where we check + if a field type is a keyword field, and if so then extract the normalizer rather + than pulling the index-time analyzer. However, a keyword normalizer is + really just a special case of an analyzer, so we should be able to simplify this + by setting the normalizer as the index-time analyzer at construction time. + +commit 809ed3b721f8e86ddffb65e00eb6baba234bb53f +Author: Nhat Nguyen +Date: Fri May 10 08:50:55 2019 -0400 + + shouldRollGeneration should execute under read lock (#41696) + + Translog#shouldRollGeneration should execute under the read lock since + it accesses the current writer. + +commit 2a8a64d3f1c605b8694f0c8eceae4fc05f61bac8 +Author: David Turner +Date: Fri May 10 14:02:09 2019 +0100 + + Remove extra `ms` from log message (#42068) + + This log message logs a `TimeValue` which includes units, but also logs an + extra `ms`. This commit removes the extra `ms`. + +commit 1be5bb5bfde5a97f6618d1e61298426f57390065 +Author: David Turner +Date: Fri May 10 13:55:27 2019 +0100 + + Recognise direct buffers in heap size docs (#42070) + + This commit slightly reworks the recommendations in the docs about setting the + heap size: + + * the "rules of thumb" are actually instructions that should be followed + + * the reason for setting `Xmx` to 50% of the heap size is more subtle than just + leaving space for the filesystem cache + + * it is normal to see Elasticsearch using more memory than `Xmx` + + * replace `cutoff` and `limit` with `threshold` since all three terms are used + interchangeably + + * since we recommend setting `Xmx` equal to `Xms`, avoid talking about setting + `Xmx` in isolation + + Relates #41954 + +commit db8fe1de003c031e5efaa32ddc6454dfb366732e +Author: Alpar Torok +Date: Fri May 10 14:24:42 2019 +0300 + + Fix slow sync test clustres artifacts task (#42012) + + * Fix slow sync test clustres artifacts task + + The task was mistakenly adding a combinational explosion of task + actions all doing the same thing. + With this PR this is fixed and each version - distribution pair is only + extracted once. + I appologieze for the SSD wear. + + * Look for configurations on the root project + * Add dependency on configurations + * This should be a `copy` so we don't blow away all the other distros + * Don't copy example plugin build directory in integration tests + +commit ea7db2bb6ad6bf91906a3deeb1f4fbb304618038 +Author: Armin Braun +Date: Fri May 10 11:59:20 2019 +0200 + + Fix testCloseOrDeleteIndexDuringSnapshot (#42007) + + * This test was resulting in a `PARTIAL` instead of a `SUCCESS` state for + the case of closing an index during snapshotting on 7.x + * The reason for this is the changed default behaviour regarding + waiting for active shards between 8.0 and 7.x + * Fixed by adjusting the waiting behaviour on the close index request + in the test + * Closes #39828 + +commit dc444cef493f405883e58aa2a2b61d24c1e0e96d +Author: Armin Braun +Date: Fri May 10 09:29:27 2019 +0200 + + Fix Race in Closing IndicesService.CacheCleaner (#42016) (#42052) + + * When close becomes true while the management pool is shut down, we run + into an unhandled `EsRejectedExecutionException` that fails tests + * Found this while trying to reproduce #32506 + * Running the IndexStatsIT in a loop is a way of reproducing this + +commit 906999f1b61d8d0213352acbc0ebf9be36c9cafa +Author: Ryan Ernst +Date: Tue May 7 22:50:11 2019 -0700 + + Disable rhel8 in packaging tests (#41924) + + This commit disables rhel 8 from being tested in vagrant packaging + tests. The vagrant image we use is beta release, but RHEL 8 was just + released, which has caused the package mirrors for the beta to stop + working. + +commit 0e3617e0ee7f7718fd4079dd5d67c3667bc3dee0 +Author: Alpar Torok +Date: Thu May 9 17:33:49 2019 +0300 + + mute failing test + + Tracked in #41256 + +commit 46e0fa2dba953cd9bf9382e449abf34345f26d0e +Author: Ryan Ernst +Date: Thu May 9 22:12:45 2019 -0700 + + Improve jdk download tests (#42034) + + This commit reworks the tests for jdk download to test the old and new + url pattern from oracle. Additionally it limits to one repository + created per version, based on the old or new pattern, and restricts + other repositories from trying to resolve jdks. + + closes #41998 + +commit 564019763282488dfdcb4ef579c1b8b9494c1e11 +Author: Tal Levy +Date: Thu May 9 22:08:31 2019 -0700 + + Refactor TransportSingleShardAction to serialize Writeable responses (#41985) (#42040) + + Previously, TransportSingleShardAction required constructing a new + empty response object. This response object's Streamable readFrom + was used. As part of the migration to Writeable, the interface here + was updated to leverage Writeable.Reader. + + relates to #34389. + +commit 99a50ac3b7cd0072c4b6e216286e02e1b670b109 +Author: Christian Mesh +Date: Thu May 9 18:14:23 2019 -0400 + + Add painless string split function (splitOnToken) (#39772) + + Adds two String split functions to Painless that can be used without enabling regexes. + +commit b23b06ddedd72e9b8636be2f4bd430ea647f4bf6 +Author: Benjamin Trent +Date: Thu May 9 14:29:10 2019 -0500 + + [ML] verify that there are no duplicate leaf fields in aggs (#41895) (#42025) + + * [ML] verify that there are no duplicate leaf fields in aggs + + * addressing pr comments + + * addressing PR comments + + * optmizing duplication check + +commit bf5a40c75467cb87eeebad0715bdacf79f98b1e2 +Author: Benjamin Trent +Date: Thu May 9 14:28:50 2019 -0500 + + [ML] relax set upgrade mode test to match what is guaranteed (#41958) (#41979) + + * [ML] relax set upgrade mode test to match what is guaranteed + + * removing unused import + +commit 57a4614a7b538f887102f371f2901179288bdeb1 +Author: Martijn van Groningen +Date: Thu May 9 21:24:06 2019 +0200 + + Keep track of the enrich key field in the enrich index. (#42022) + + The enrich key field is being kept track in _meta field by the policy runner. + The ingest processor uses the field name defined in enrich index _meta field and + not in the policy. This will avoid problems if policy is changed without + a new enrich index being created. + + This also complete decouples EnrichPolicy from ExactMatchProcessor. + + The following scenario results in failure without this change: + 1) Create policy + 2) Execute policy + 3) Create pipeline with enrich processor + 4) Use pipeline + 5) Update enrich key in policy + 6) Use pipeline, which then fails. + +commit 732ef15f0dbd8db572cdff75a54cfed1b0e09d9e +Author: James Rodewig +Date: Thu May 9 13:10:19 2019 -0400 + + [DOCS] Adds placeholder for 7.1.0 release notes (#42024) + +commit 299ff70bfe5f4d24fe62db43485a2fa9dabcb849 +Author: Martijn van Groningen +Date: Thu May 9 18:14:01 2019 +0200 + + Enrich store should only update the policies via an update task. (#41944) + +commit 2998c107fbfd06f1ffe563e4357438045ff19c02 +Author: Jay Modi +Date: Thu May 9 09:41:23 2019 -0600 + + Fix node close stopwatch usage (#41918) + + The close method in Node uses a StopWatch to time to closing of + various services. However, the call to log the timing was made before + any of the services had been closed and therefore no timing would be + printed out. This change moves the timing log call to be a closeable + that is the last item closed. + +commit 9284a70ec8905225731712b76589189ab0d08ce1 +Author: Lisa Cawley +Date: Thu May 9 07:48:23 2019 -0700 + + [DOCS] Updates security configuration overview (#41982) + +commit 598e0962ed5fce0034a5261a823d446d57e4d9da +Author: Jason Tedor +Date: Thu May 9 10:23:36 2019 -0400 + + Fix compilation in JVMErgonomicsTests + + This issue arose after a cherry-pick from the wrong branch. Sorry. This + commit addresses the issue. + +commit f3bcc4fc225045ee798b76946b385435845f9aa0 +Author: Jay Modi +Date: Thu May 9 08:02:43 2019 -0600 + + Default seed address tests account for no IPv6 (#41971) + + This change makes the default seed address tests account for the lack + of an IPv6 network. By default docker containers only run with IPv4 and + these tests fail in a vanilla installation of elasticsearch-ci. To + resolve this we only expect IPv6 seed addresses if IPv6 is available. + + Relates #41404 + +commit 970a2254c3c623dc92fa2e19c6956606e4a128d4 +Author: Jason Tedor +Date: Thu May 9 10:10:27 2019 -0400 + + Limit max direct memory size to half of heap size (#42006) + + This commit adds an ergonomic choice ot the max direct memory size such + that if it is not set, we default it to half of the heap size. + +commit ea5019665a8847a26f03bae901dc4b7f57e80fa6 +Author: James Rodewig +Date: Thu May 9 09:51:12 2019 -0400 + + [DOCS] Replace table with def list for ids query (#41865) + +commit 7ee541546a7a7db88028b526355120f0c5153a80 +Author: Jason Tedor +Date: Thu May 9 09:19:30 2019 -0400 + + Account for Java 8 in JVM options parsing + + Java 8 presents the JVM options slightly differently when displaying via + -XX:+PrintFlagsFinal. This commit adapts the JVM options parser for this + possibility. + + Relates #42009 + +commit 0b21fb0ee6ac99a52e4a33ce23cfd59db5c43676 +Author: Daniel Schneiter +Date: Thu May 9 15:08:31 2019 +0200 + + Mentioned the name of the icu_analyzer + +commit 256588d7735fdba4cd94975deb186c6eb9c74387 +Author: David Kyle +Date: Thu May 9 13:46:59 2019 +0100 + + Mute IndexStatsIT#testFilterCacheStats + + See https://github.com/elastic/elasticsearch/issues/32506 + +commit b7c7ca8f0946c73a1226855d2e8953069f6387d2 +Author: Jim Ferenczi +Date: Thu May 9 14:25:07 2019 +0200 + + Fix IAE on cross_fields query introduced in 7.0.1 (#41938) + + If the max doc in the index is greater than the minimum total term frequency + among the requested fields we need to adjust max doc to be equal to the min ttf. + This was removed by mistake when fixing #41125. + + Closes #41934 + +commit c0d67919c8472337ca01da59aa14d64deea9eeef +Author: David Kyle +Date: Thu May 9 13:16:13 2019 +0100 + + Mute ApiKeyIntegTests + + See https://github.com/elastic/elasticsearch/issues/41747 + +commit 309e4a11b5060e290e6b2ab0bce6e61a421b5cd5 +Author: Alan Woodward +Date: Thu May 9 13:08:33 2019 +0100 + + Cut AnalyzeResponse over to Writeable (#41915) + + This commit makes AnalyzeResponse and its various helper classes implement + Writeable. The classes are also now immutable. + + Relates to #34389 + +commit 44f09a9a86953b4c9ccd9889b860868d6161dca4 +Merge: 202a840da96 37771502aec +Author: Martijn van Groningen +Date: Thu May 9 13:33:34 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 8e33a5292aa73380f1f6cb9359368783e50f1715 +Author: Alexander Reelsen +Date: Thu May 9 12:59:45 2019 +0200 + + Add HTML strip processor (#41888) + + This processor uses the lucene HTMLStripCharFilter class to remove HTML + entities from a field. This adds to the char filter, so that there is + possibility to store the stripped version as well. + + Note, that the characeter filter replaces tags with a newline, so that + the produced HTML will look slightly different than the incoming HTML + with regards to newlines. + +commit 37771502aecf24c9e429d56f86260587c24519db +Author: Jason Tedor +Date: Thu May 9 06:30:46 2019 -0400 + + Remove manual parsing of JVM options (#41962) + + This commit removes manual parsing of JVM options when calculating + ergonomics. This is to avoid a situation that we parse values + differently than the JVM would. In fact, we already have a bug along + these lines today. It is possible to start the JVM with the same flag + multiple times on the command line. In this case, the last value + wins. For example, -Xmx1g -Xmx2g would start the JVM with a heap size of + two gigabytes. Our JVM ergonomics ignores this possibility and instead + the first value is winning! + + Our strategy to avoid manual parsing of the JVM options is to start the + Java command line parser (without actually starting a JVM) by invoking + java with the same command line flags as presented and request that the + JVM tell us what values it would start with. This ensures that we have + the correct values when making ergonomic decisions. + + Moreover, our strategy also is ignoring ES_JAVA_OPTS which could + override the heap size as well leading to incorrect ergonomic + choices. This commit address this issue too. + +commit a48b402e907e9d041adbc6b87e218731c7c65556 +Author: Alpar Torok +Date: Thu May 9 10:12:30 2019 +0300 + + Upgrade to Gradle 5.4.1 (#41750) + + * Upgrade to Gradle 5.4.1 + + https://docs.gradle.org/5.4/release-notes.html + + Notable: Support for JDK12 , API for incremental tasks + + * Use newer version of checkstyle + + * Increase stack size + +commit a329aaec9069d76d699dd634e555f484177c5d48 +Author: Jim Ferenczi +Date: Thu May 9 08:46:34 2019 +0200 + + Fix assertion error when caching the result of a search in a read-only index (#41900) + + The ReadOnlyEngine wraps its reader with a SoftDeletesDirectoryReaderWrapper if soft deletes + are enabled. However the wrapping is done on top of the ElasticsearchDirectoryReader and that + trips assertion later on since the cache key of these directories are different. This commit + changes the order of the wrapping to put the ElasticsearchDirectoryReader first in order to + ensure that it is always retrieved first when we unwrap the directory. + + Closes #41795 + +commit 83fef23fd137838f23677537dcbe386baee7fa1f +Author: Flavio Pompermaier +Date: Thu May 9 08:52:07 2019 +0200 + + Fix wrong property name (#40636) + +commit 4358cc6ac81793636be8f9208446a996ff0d2dd6 +Author: Gordon Brown +Date: Wed May 8 16:42:50 2019 -0600 + + Add note about ILM action ordering (#41771) + + Adds a note clarifying that actions are ordered automatically. + +commit edd6438e3467e8ee2e5a0162d83982bfe10428a9 +Author: Benjamin Trent +Date: Wed May 8 15:03:28 2019 -0500 + + mute test related to #41967 (#41968) + +commit 6343ec3d3e2eaeff945011f3ba5f2756d9974fa3 +Author: Ryan Ernst +Date: Wed May 8 12:09:02 2019 -0700 + + Update lintian overrides (#41561) (#41953) + + The deb package has been updated several times in the past to contain + overrides in order to pass lintian inspection. However, there have never + been any tests to ensure we do not fallback to failure. This commit + updates the overrides file given things that have changed since 2.x like + adding ML and bundling the jdk. + + closes #17185 + +commit 202a840da9640f05efe45f430928c11663dafa23 +Merge: 1b00e7f834b 2c561481cd2 +Author: Michael Basnight +Date: Wed May 8 13:59:01 2019 -0500 + + Merge remote-tracking branch 'upstream/7.x' into enrich-7.x + +commit 2c561481cd274e523c1fa971bb7138a26e974550 +Author: Jack Conradson +Date: Wed May 8 10:29:02 2019 -0700 + + Add static section whitelist info to api docs generation (#41870) + + This change adds imported methods, class bindings, and instance bindings to the documentation generation for the Painless Context APIs. + +commit a2b7871f9fd0f6faf627fd45218fcb137d034a43 +Author: William Brafford +Date: Wed May 8 14:15:22 2019 -0400 + + Allow unknown task time in QueueResizingEsTPE (#41957) + + * Allow unknown task time in QueueResizingEsTPE + + The afterExecute method previously asserted that a TimedRunnable task + must have a positive execution time. However, the code in TimedRunnable + returns a value of -1 when a task time is unknown. Here, we expand the + logic in the assertion to allow for that possibility, and we don't + update our task time average if the value is negative. + + * Add a failure flag to TimedRunnable + + In order to be sure that a task has an execution time of -1 because of + a failure, I'm adding a failure flag boolean to the TimedRunnable class. + If execution time is negative for some other reason, an assertion will + fail. + + Backport of #41810 + Fixes #41448 + +commit f410f91f135f640838b2de56753b20cb5528f420 +Author: Zachary Tong +Date: Wed May 8 12:37:43 2019 -0400 + + Cleanup RollupSearch exceptions, disallow partial results (#41272) + + - msearch exceptions should be thrown directly instead of wrapping + in a RuntimeException + - Do not allow partial results (where some indices are missing), + instead throw an exception if any index is missing + +commit 374ce3e6a80818221c5c80a8a266fc42cad62f0c +Author: Ryan Ernst +Date: Wed May 8 07:16:13 2019 -0700 + + Add gradle plugin for downloading jdk (#41461) + + We currently download 3 variants of the same version of the jdk for + bundling into the distributions. Additionally, the vagrant images do + their own downloading. This commit moves the jdk downloading into a + utility gradle plugin. This will be used in a future PR by the packaging + tests. + + The new plugin exposes a "jdks" project extension which allows creating + named jdks. Once the jdk version and platform are set for a named jdk, + the jdk object may be used as a lazy String for the jdk home path, or a + file collection for copying. + +commit 7841fa4814866e1ca4c0f2c65ff214200a6f5bc2 +Author: Alpar Torok +Date: Wed May 8 16:31:51 2019 +0300 + + Add corretto 11 and 8 to runtime javas (#41946) + + We now have coretto on the ephemeral CI images so we can add it to our + regular java test matrix. + On backporting will also add corretto8 to branches that support it. + +commit 452ee55cdb297bacebc9cecaf3d7083a3f077fa3 +Author: David Roberts +Date: Wed May 8 13:46:58 2019 +0100 + + Make ISO8601 date parser accept timezone when time does not have seconds (#41896) + + Prior to this change the ISO8601 date parser would only + parse an optional timezone if seconds were specified. + This change moves the timezone to the same level of + optional components as hour, so that timestamps without + minutes or seconds may optionally contain a timezone. + It also adds a unit test to cover all the supported + formats. + +commit 957046dad0c40ce26027746c910da32f3fd3591a +Author: Yannick Welsch +Date: Wed May 8 13:57:01 2019 +0200 + + Allow IDEA test runner to control number of test iterations (#41653) + + Allows configuring the number of test iterations via IntelliJ's config dialog, instead of having to add it + manually via the tests.iters system property. + +commit ba9d2ccc1f7ea8ace27c650fbc63e5460850ed5f +Author: David Kyle +Date: Wed May 8 12:23:49 2019 +0100 + + [ML Data Frame] Set executing nodes in task actions (#41798) + + Direct the task request to the node executing the task and also refactor the task responses + so all errors are returned and set the HTTP status code based on presence of errors. + +commit 60f84a2eb2bc2309b2bb4094ceaf7fb4fcb9b657 +Author: David Turner +Date: Wed May 8 12:08:47 2019 +0100 + + Remove mention of bulk threadpool in examples (#41935) + + The `bulk` threadpool is now called `write`, but `bulk` is still + used in some examples. This commit fixes that. + + Also, the only way `threadpool.bulk.write: 30` is a valid increase in the size + of this threadpool is if you have 29 processors, which is an odd number of + processors to have. This commit removes the "more threads" bit. + +commit 5c824f39936b4a33c70fe0bab4b96162c9f22438 +Author: Armin Braun +Date: Wed May 8 13:10:19 2019 +0200 + + Reenable testCloseOrDeleteIndexDuringSnapshot (#41892) + + * Relates #39828 + +commit 711ace0533257caae09032b7aff9b3f6806ccb63 +Author: Alpar Torok +Date: Wed May 8 14:00:11 2019 +0300 + + Testclusters: support for security and convert example plugins (#41864) + + testclusters detect from settings that security is enabled + if a user is not specified using the DSL introduced in this PR, a default one is created + the appropriate wait conditions are used authenticating with the first user defined in the DSL ( or the default user ). + an example DSL to create a user is user username:"test_user" password:"x-pack-test-password" role: "superuser" all keys are optional and default to the values shown in this example + +commit ca3d8817165d5b980f3dbb72647ce77cfab936f1 +Author: Jim Ferenczi +Date: Wed May 8 12:25:58 2019 +0200 + + Always set terminated_early if terminate_after is set in the search request (#40839) + + * terminated_early should always be set in the response with terminate_after + + Today we set `terminated_early` to true in the response if the query terminated + early due to `terminate_after`. However if `terminate_after` is smaller than + the number of documents in a shard we don't set the flag in the response indicating + that the query was exhaustive. This change fixes this disprepancy by setting + terminated_early to false in the response if the number of documents that match + the query is smaller than the provided `terminate_after` value. + + Closes #33949 + +commit 99b5a27ea0072d6f12ec6181989f1cdc365c9f3e +Author: David Turner +Date: Wed May 8 10:23:55 2019 +0100 + + Node names in bootstrap config have no ports (#41569) + + In cases where node names and transport addresses can be muddled, it is unclear + that `cluster.initial_master_nodes: master-a:9300` means to look for a node + called `master-a:9300` rather than a node called `master-a` with transport port + `9300`. This commit adds docs to that effect. + +commit 818e05c05f48558085cc4a94f7156746d94a068f +Author: Yannick Welsch +Date: Wed May 8 09:28:27 2019 +0200 + + Highlight the use of single-node discovery in docker docs (#41241) + + Relates to https://discuss.elastic.co/t/es-7-and-docker-single-node-cluster/176585 + +commit 785ae091014b918cd10339f3f8fb685a6f9821c9 +Author: Yannick Welsch +Date: Wed May 8 09:27:06 2019 +0200 + + Allow reindexing into write alias (#41677) + + Fixes an issue where reindex currently fails if the destination is an alias pointing to multiple indices, + even it is using a write index. + + Closes #41667 + +commit 5b71baa10066287f52665cbd6b0591abcd2034b6 +Author: Yannick Welsch +Date: Wed May 8 09:26:37 2019 +0200 + + Upgrade SDK and test discovery-ec2 credential providers (#41732) + + Upgrades the AWS SDK to the same version that we're using for the repository-s3 plugin, providing + testing capabilities to override certain SDK endpoints in order to point them to localhost for testing. + Adds tests for the various credential providers. + +commit 4c909e93bbff96f2102d93f3154d5bb51c710db9 +Author: David Turner +Date: Wed May 8 08:34:32 2019 +0100 + + Reject port ranges in `discovery.seed_hosts` (#41905) + + Today Elasticsearch accepts, but silently ignores, port ranges in the + `discovery.seed_hosts` setting: + + ``` + discovery.seed_hosts: 10.1.2.3:9300-9400 + ``` + + Silently ignoring part of a setting like this is trappy. With this change we + reject seed host addresses of this form. + + Closes #40786 + Backport of #41404 + +commit e04953a2bf1a145ecd0b55393c44b30597fae4f4 +Author: Tim Vernum +Date: Wed May 8 16:07:21 2019 +1000 + + Clarify settings in default SSL/TLS (#41930) + + The settings listed under the "Default values for TLS/SSL settings" + heading are not actual settings, rather they are common suffixes that + are used for settings that exist in a variety of contexts. + + This commit changes the way they are presented to reduce this + confusion. + + Backport of: #41779 + +commit 8bea3c3a582d980aac31d0717c9ad2ae8e340af8 +Author: Jason Tedor +Date: Tue May 7 22:44:47 2019 -0400 + + Enable trace logging in CCR retention lease tests + + These tests are failing somewhat mysteriously, indicating that when we + renew retention leaess during a restore that our retention leases that + we added before starting the restore suddenly do not exist. To make + sense of this, this commit enables trace logging. + +commit d5b0badeb7510bb6e1cb0f43606b942ab57ac06b +Author: Marios Trivyzas +Date: Tue May 7 23:09:24 2019 +0300 + + SQL: Remove CircuitBreaker from parser (#41835) + + The CircuitBreaker was introduced as means of preventing a + `StackOverflowException` during the build of the AST by the parser. + + The ANTLR4 grammar causes a weird behaviour for a Parser Listener. + The `enterEveryRule()` method is often called with a different parsing + context than the respective `exitEveryRule()`. This makes it difficult + to keep track of the tree's depth, and a custom Map was used as an + attempt of matching the contextes as they are encounter during `enter` + and during `exit` of the rules. + + This approach had 2 important drawbacks: + 1. It's hard to maintain this custom Map as the grammar changes. + 2. The CircuitBreaker could often lead to false positives which caused + valid queries to return an Exception and prevent them from executing. + + So, this removes completely the CircuitBreaker which is replaced be + a simple handling of the `StackOverflowException` + + Fixes: #41471 + (cherry picked from commit 1559a8e2dbd729138b52e89b7e80264c9f4ad1e7) + +commit cf8a2be27bf2f3d7326bae7471f599d812c3aceb +Author: Lisa Cawley +Date: Tue May 7 10:03:06 2019 -0700 + + [DOCS] Fix callouts for dataframe APIs (#41904) + +commit 935f70c05e38957180498aee679a8036bf5e6bf0 +Author: David Turner +Date: Tue May 7 17:39:07 2019 +0100 + + Handle serialization exceptions during publication (#41781) + + Today if an exception is thrown when serializing a cluster state during + publication then the master enters a poisoned state where it cannot publish any + more cluster states, but nor does it stand down as master, yielding repeated + exceptions of the following form: + + ``` + failed to commit cluster state version [12345] + org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publishing failed + at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1045) ~[elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:252) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:238) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:142) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.0.0.jar:7.0.0] + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_144] + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_144] + at java.lang.Thread.run(Thread.java:748) [?:1.8.0_144] + Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: cannot start publishing next value before accepting previous one + at org.elasticsearch.cluster.coordination.CoordinationState.handleClientValue(CoordinationState.java:280) ~[elasticsearch-7.0.0.jar:7.0.0] + at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1030) ~[elasticsearch-7.0.0.jar:7.0.0] + ... 11 more + ``` + + This is because it already created the publication request using + `CoordinationState#handleClientValue()` but then it fails before accepting it. + This commit addresses this by performing the serialization before calling + `handleClientValue()`. + + Relates #41090, which was the source of such a serialization exception. + +commit 4cca1e8fff52fac11567c64399d8301b0faa9871 +Author: Alan Woodward +Date: Tue May 7 17:26:39 2019 +0100 + + Correct spelling of MockLogAppender.PatternSeenEventExpectation (#41893) + + The class was called PatternSeenEventExcpectation. This commit + is a straight class rename to correct the spelling. + +commit e9e4bae6833f192705bdbc5806c1b328524aa320 +Author: Ryan Ernst +Date: Tue May 7 09:08:15 2019 -0700 + + Fix fractional seconds for strict_date_optional_time (#41871) + + The fractional seconds portion of strict_date_optional_time was + accidentally copied from the printer, which always prints at least 3 + fractional digits. This commit fixes the formatter to allow 1 or 2 + fractional seconds. + + closes #41633 + +commit 58041f3fdb0194d54b3d6f76c2bf42ad8d20efbf +Author: Ioannis Kakavas +Date: Tue May 7 18:59:24 2019 +0300 + + Remove op.name configuration setting (#41445) + + This setting was not eventually used in the realm and thus can be + removed + +commit 8823cb65f7510f3453f034ca29a4365b5c78de80 +Author: Hendrik Muhs +Date: Tue May 7 16:14:35 2019 +0200 + + [ML-DataFrame] migrate to PageParams for get and stats, move PageParams into core (#41851) + + migrate hlrc dataframe get and _stats to use PageParams, moves PageParams into core for common + usage, fix possible NPE in PageParams + +commit 77f634ba2587efa01114d69c61a5e9645e801321 +Author: James Rodewig +Date: Tue May 7 09:22:59 2019 -0400 + + [DOCS] Rewrite `exists` query docs (#41868) + +commit f5bb1cc65a444b8383590453fb8728a91ae9834e +Author: James Rodewig +Date: Tue May 7 09:06:43 2019 -0400 + + [DOCS] Fix typo referring to multi search API + +commit f068a22f5f48c7bc3f897221ed47063461ee20be +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Apr 29 19:56:16 2019 +0200 + + SeqNo CAS linearizability (#38561) + + Add a test that stresses concurrent writes using ifSeqno/ifPrimaryTerm to do CAS style updates. Use linearizability checker to verify linearizability. Linearizability of successful CAS'es is guaranteed. + + Changed linearizability checker to allow collecting history concurrently. + + Changed unresponsive network simulation to wake up immediately when network disruption is cleared to ensure tests proceed in a timely manner (and this also seems more likely to provoke issues). + +commit 1b00e7f834b3227737c25c2228decd49f4640a5a +Author: Martijn van Groningen +Date: Tue May 7 11:07:46 2019 +0200 + + Change the reindex fetch in policy runner from 1000 to 10000 and (#41838) + + Reindex uses scroll searches to read the source data. It is more efficient + to read more data in one search scroll round then several. I think 10000 + is a good sweet spot. + + Relates to #32789 + +commit d7fd51a84e726b237623e2a4e3ab37b193cf3b09 +Author: Jason Tedor +Date: Tue May 7 06:34:51 2019 -0400 + + Provide names for all artifact repositories (#41857) + + This commit adds a name for each Maven and Ivy repository used in the + build. + +commit 3a35427b6d81c0fb7fbe65aa9ac223430ec1d5ed +Author: Alan Woodward +Date: Tue May 7 09:09:28 2019 +0100 + + Improvements to docs around multiplexer and synonyms (#41645) + + This commit fixes a multiplexer doc error concerning synonyms, and adds + suggestions on how to combine the two filters. + +commit d709b8bb97c7c60139dd2bb37bd605160e588598 +Author: Martijn van Groningen +Date: Tue May 7 08:38:08 2019 +0200 + + Rename enrich policy index_pattern field to indices. (#41836) + + Relates to #32789 + +commit f366f56f001ce1c7d35b90f351a596037e59280e +Author: Martijn van Groningen +Date: Tue May 7 08:37:48 2019 +0200 + + Change policy runner to use helper method on EnrichPolicy instead of (#41839) + + its own helper method to determine alias / policy base name. + + This way both the enrich processor and policy runner use the same logic + to determine the alias to use. + + Relates to #32789 + +commit c808badb23035c601e598afe079621d3657b14d6 +Author: Jason Tedor +Date: Mon May 6 21:04:13 2019 -0400 + + Add tasks to build Docker build context artifacts (#41819) + + This commit adds some tasks to generate dedicated Docker build context + artifacts. + +commit 3508b6c641e9f984d9612b820574fa52fce1ecb6 +Author: Tim Vernum +Date: Tue May 7 09:55:48 2019 +1000 + + Log warning when unlicensed realms are skipped (#41828) + + Because realms are configured at node startup, but license levels can + change dynamically, it is possible to have a running node that has a + particular realm type configured, but that realm is not permitted under + the current license. + In this case the realm is silently ignored during authentication. + + This commit adds a warning in the elasticsearch logs if authentication + fails, and there are realms that have been skipped due to licensing. + This message is not intended to imply that the realms could (or would) + have successfully authenticated the user, but they may help reduce + confusion about why authentication failed if the caller was expecting + the authentication to be handled by a particular realm that is in fact + unlicensed. + + Backport of: #41778 + +commit a90aac1c711a7102b2983c96f8946236d330b170 +Author: Julie Tibshirani +Date: Mon May 6 11:52:22 2019 -0700 + + Clarify that path_match also considers object fields. (#41658) + + The `path_match` and `path_unmatch` parameters in dynamic templates match on + object fields in addition to leaf fields. This is not obvious and can cause + surprising errors when a template is meant for a leaf field, but there are + object fields that match. This PR adds a note to the docs to describe the + current behavior. + +commit 6fd8924c5a8d5151820c30926d131d3bb02341b1 +Author: Ryan Ernst +Date: Mon May 6 12:32:31 2019 -0700 + + Switch run task to use real distro (#41590) + + The run task is supposed to run elasticsearch with the given plugin or + module. However, for modules, this is most realistic if using the full + distribution. This commit changes the run setup to use the default or + oss as appropriate. + +commit 2cdd75e2c8f766a48c9ad84208bc6f312ed190ce +Author: Issam EL-ATIF <900456+ielatif@users.noreply.github.com> +Date: Mon May 6 20:23:03 2019 +0200 + + [DOCS] remove 'es.scripting.update.ctx_in_params' system property for 7.0 (#41643) + +commit eb9bce3930c7c674375a5d363702d3c716970ba8 +Author: Julie Tibshirani +Date: Mon May 6 10:38:54 2019 -0700 + + Clarify _doc is a permanent part of certain document APIs. (#41727) + + We received some feedback that it is not completely clear why `_doc` is present + in the typeless document APIs: + + > The new index APIs are PUT {index}/_doc/{id} in case of explicit ids and POST + {index}/_doc for auto-generated ids."_ Isn't this contradicting? Specifying + *types in requests is deprecated*, but we are supposed to still mention *_doc* + in write requests? + + This PR updates the 'removal of types' documentation to try to clarify that + `_doc` now represents the endpoint name, as opposed to a type. + +commit 70bf432fa80d10a4aab5b16a0e18ffc02c2bcfc0 +Author: Jim Ferenczi +Date: Mon May 6 18:45:35 2019 +0200 + + Fix full text queries test that start with now (#41854) + + Full text queries that start with now are not cacheable if they target a date field. + However we assume in the query builder tests that all queries are cacheable and this assumption + fails when the random generated query string starts with "now". This fails twice in several years + since the probability that a random string starts with "now" is low but this commit ensures that + isCacheable is correctly checked for full text queries that fall into this edge case. + + Closes #41847 + +commit e0e8638267b21d460637d36fcb9c3407bcd532e7 +Author: Alpar Torok +Date: Mon May 6 19:53:32 2019 +0300 + + Remove the jdk directory to save space on bwc tests (#41743) + + * Revert "Revert "Clean up clusters between tests (#41187)"" + + This reverts commit 9efc853aa668e285ede733d37b6fc7a0f4b02041. + + * Remove the jdk directory to save space on bwc tests + + This PR adresses the same concern as #41187 in a different way. + It removes only the JDK from the distribution once the cluster stops, + so we keep the same disk space requirements as before adding the JDK. + This is still a temporary measure, testclusters already deals with this + by doing the equivalent of `cp -l` instead of an actual copy. + +commit 50fc27e9a00484c5d9e25d21b3d626c1cd7d7ab4 +Author: Benjamin Trent +Date: Mon May 6 10:56:26 2019 -0500 + + [ML] addresses preview bug, and adds check to PUT (#41803) (#41850) + +commit 79b7ce86975cf80068216adfcb19e0f1bb8bd6f9 +Author: Przemyslaw Gomulka +Date: Mon May 6 17:55:11 2019 +0200 + + Fix javadoc in WrapperQueryBuilder backport(41641) #41849 + + missing brackets in javadoc + backports #41641 + +commit 927013426a1fca0606c75d94c18cb07928f90243 +Author: Tim Brooks +Date: Mon May 6 09:51:32 2019 -0600 + + Read multiple TLS packets in one read call (#41820) + + This is related to #27260. Currently we have a single read buffer that + is no larger than a single TLS packet. This prevents us from reading + multiple TLS packets in a single socket read call. This commit modifies + our TLS work to support reading similar to the plaintext case. The data + will be copied to a (potentially) recycled TLS packet-sized buffer for + interaction with the SSLEngine. + +commit 228d23de6d7fda72aa02a8f06da68caf5faca80a +Author: Marios Trivyzas +Date: Mon May 6 18:00:50 2019 +0300 + + SQL: [Docs] Add example for custom bucketing with CASE (#41787) + + Add a TIP on how to use CASE to achieve custom bucketing + with GROUP BY. + + Follows: #41349 + + (cherry picked from commit eb5f5d45533c5f81e57dd0221d902a73ec400098) + +commit 2306531815cf0d0af01b2baf04061ec31e57eaac +Author: James Rodewig +Date: Mon May 6 10:36:07 2019 -0400 + + [DOCS] Rewrite `term` query docs for new format (#41498) + + * [DOCS] Restructure `term` query docs. + +commit c25736c410dcafc8779bcba7987b27e6c210964d +Author: James Baiera +Date: Mon May 6 10:32:46 2019 -0400 + + Backport 7.x - Add enrich policy runner (#41088) (#41759) + + Backports #41088 + + Adds the foundation of the execution logic to execute an enrich policy. Validates + the source index existence as well as mappings, creates a new enrich index for + the policy, reindexes the source index into the new enrich index, and swaps the + enrich alias for the policy to the new index. + +commit 85800538185c9e689f927edd1da23d53bf1947c4 +Author: Daniel Mitterdorfer +Date: Mon May 6 16:15:37 2019 +0200 + + Mute PermissionsIT#testWhen[...]ByILMPolicy (#41859) + + Relates #41440 + Relates #41858 + +commit 227d5e15fb7d612b07578939a7d0131b61da33d0 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon May 6 16:06:51 2019 +0200 + + ReadOnlyEngine assertion fix (#41842) + + Fixed the assertion that maxSeqNo == globalCheckpoint to actually check + against the global checkpoint. + +commit 2455295c6dafece8de8fb162900bce64bcd1d61c +Author: Alpar Torok +Date: Mon May 6 14:49:01 2019 +0300 + + Testsclusters use seprate configurations per version (#41504) + + * artifact config tests WIP + + * wip version configuration + + * Tests for older versions + + * use separate configurations per version + + * checkstyle + + * PR review + +commit d54a92103219c65537ae5f8ee163de50c5fe3755 +Author: Hendrik Muhs +Date: Mon May 6 10:03:20 2019 +0200 + + remove unused import + +commit 0c03707704097f9a225fa19f890340dfa003d43e +Author: Hendrik Muhs +Date: Mon May 6 09:39:58 2019 +0200 + + [ML-DataFrame] reset/clear the position after indexer is done (#41736) + + reset/clear the position after indexer is done + +commit ee84038699ce22592df4154f37e2edb7b4c3e8fb +Author: Tim Vernum +Date: Mon May 6 16:40:38 2019 +1000 + + Update security acknowledgement messages for basic (#41825) + + When applying a license update, we provide "acknowledgement messages" + that indicate which features will be affected by the change in license. + + This commit updates the messages that are provided when installing a + basic license, so that they reflect the changes made to the security + features that are included in that license type. + + Backport of: #41776 + +commit ff317d0d7fee3c058f1cb830fa846f4358a55633 +Author: Jason Tedor +Date: Sun May 5 11:16:07 2019 -0400 + + Set metadata sources for Ivy repositories (#41818) + + We have faked some Ivy repositories on a few artifact locations. Today + when Gradle attempts to resolve these artifacts, it follows its default + strategy to search for Gradle metadata, then Maven POM files, then Ivy + descriptors, and finally will fallback to looking directly for the + artifact. This wastes times on remote network calls that will 404 anyway + since these metadata resources will not exist for these fake Ivy + repositories. This commit overrides the Gradle strategy to look directly + for artifacts. + +commit 4a88da70c577d489e66084a51d365dfc85f2258b +Author: Hicham Mallah +Date: Sun May 5 02:10:29 2019 +0300 + + Add index name to cluster block exception (#41489) + + Updates the error message to reveal the index name that is causing it. + + Closes #40870 + +commit 793f13c8b6f1a062da005f5289e6b30d02a70693 +Author: Jason Tedor +Date: Sat May 4 12:48:04 2019 -0400 + + Enforce that Maven/Ivy repositories use https (#41812) + + This commit adds a check when a repository is added that we are using + https instead of http. + +commit f4da98ca3d34aeafcf3e05683fc9f34701de842d +Author: Jason Tedor +Date: Sat May 4 12:43:14 2019 -0400 + + Use a proper repository for ml-cpp artifacts (#41817) + + This switches the strategy used to download machine learning artifacts + from a manual download through S3 to using an Ivy repository on top of + S3. This gives us all the benefits of Gradle dependency resolution + including local caching. + +commit 8df13b474d0dfb082ca217d5bf547ab2d3391017 +Author: Jason Tedor +Date: Sat May 4 08:28:30 2019 -0400 + + Update some more S3 artifact locations to use https + + This commit updates some additional S3 artifact locations to use https + instead of http. + + Relates 241c4ef97a34b74f2d8c13b52d60e74b5adba46b + +commit b69e28177b902c3f1829dc2280a3d7e24ab57a25 +Author: Benjamin Trent +Date: Fri May 3 18:18:27 2019 -0500 + + [ML] rewriting stats gathering to use callbacks instead of a latch (#41793) (#41804) + +commit c7924014fad6ba9ba3191d92f5e820c12b60994f +Author: Nhat Nguyen +Date: Fri May 3 18:47:14 2019 -0400 + + Verify consistency of version and source in disruption tests (#41614) (#41661) + + With this change, we will verify the consistency of version and source + (besides id, seq_no, and term) of live documents between shard copies + at the end of disruption tests. + +commit 24484ae227dc89ecec8692145c9fcd174130e298 +Author: Tim Brooks +Date: Fri May 3 13:43:48 2019 -0600 + + Fix http read timeout test by releasing response (#41801) + + This fixes #41794. Currently the read timeout test queues up responses + in the netty pipeline. These responses are immediately returned in the + write call, but they are not released. This commit releases the + responses. This will cause the leak detector to quit throwing + exceptions. + +commit 241c4ef97a34b74f2d8c13b52d60e74b5adba46b +Author: Jason Tedor +Date: Fri May 3 16:13:34 2019 -0400 + + Use https for artifact locations + + This commit switches to using https for some artifact locations. + +commit e61469aae667858c23b3187d5115704585e1ac8e +Author: Nhat Nguyen +Date: Fri May 3 11:39:16 2019 -0400 + + Noop peer recoveries on closed index (#41400) + + If users close an index to change some non-dynamic index settings, then the current implementation forces replicas of that closed index to copy over segment files from the primary. With this change, we make peer recoveries of closed index skip both phases. + + Relates #33888 + + Co-authored-by: Yannick Welsch + +commit 23706d4cdf72dc80001d760f413569cf8f89d3ce +Author: Issam EL-ATIF <900456+ielatif@users.noreply.github.com> +Date: Fri May 3 17:54:22 2019 +0200 + + Update error message for allowed characters in aggregation names (#41573) + + Exception message thrown when specifying illegal characters did + no accurately described the allowed characters. This updates the + error message to reflect reality (any character except [, ] and >) + +commit 4421744184f435fd456d58196acba4846a401f04 +Author: David Kyle +Date: Fri May 3 16:30:41 2019 +0100 + + [ML DataFrame] Update Preview docs for source config change (#41761) + +commit 9a4cab9f77ec4e96b11e3e9f48bd0ff0261e0aa8 +Author: Gordon Brown +Date: Fri May 3 09:10:29 2019 -0600 + + Recommend use of ln1p/log1p to avoid negative scores (#41610) + + As negative scores will now cause an error, and it is easy to + accidentally produce negative scores with some of the built-in modifiers + (especially `ln` and `log`), this adjusts the documentation to more + strongly recommend the use of `ln1p` and `log1p` instead. + + Also corrects some awkward formatting on the note sections following the + table. + +commit 03c959f18827bdad53e261c1036791463e098633 +Author: Jason Tedor +Date: Fri May 3 10:28:54 2019 -0400 + + Upgrade keystore on package install (#41755) + + When Elasticsearch is run from a package installation, the running + process does not have permissions to write to the keystore. This is + because of the root:root ownership of /etc/elasticsearch. This is why we + create the keystore if it does not exist during package installation. If + the keystore needs to be upgraded, that is currently done by the running + Elasticsearch process. Yet, as just mentioned, the Elasticsearch process + would not have permissions to do that during runtime. Instead, this + needs to be done during package upgrade. This commit adds an upgrade + command to the keystore CLI for this purpose, and that is invoked during + package upgrade if the keystore already exists. This ensures that we are + always on the latest keystore format before the Elasticsearch process is + invoked, and therefore no upgrade would be needed then. While this bug + has always existed, we have not heard of reports of it in practice. Yet, + this bug becomes a lot more likely with a recent change to the format of + the keystore to remove the distinction between file and string entries. + +commit 873d0020a5a6549390a9b4e4cfeccb332734729f +Author: David Turner +Date: Fri May 3 14:49:23 2019 +0200 + + Reject null customs at build time (#41782) + + Today you can add a null `Custom` to the cluster state or its metadata, but + attempting to publish such a cluster state will fail. Unfortunately, the + publication-time failure gives very little information about the source of the + problem. This change causes the failure to manifest earlier and adds + information about which `Custom` was null in order to simplify the + investigation. + + Relates #41090. + +commit 00af42fefe42c00401222d92ceb4c422d117d478 +Author: Hendrik Muhs +Date: Fri May 3 14:15:45 2019 +0200 + + move checkpoints into x-pack core and introduce base classes for data frame tests (#41783) + + move checkpoints into x-pack core and introduce base classes for data frame tests + +commit befe2a45b98cf7845789b8f666a33c0c1a6fff00 +Author: Hendrik Muhs +Date: Fri May 3 13:36:59 2019 +0200 + + [ML-DataFrame] refactor pivot to only take the pivot config (#41763) + + refactor pivot class to only take the config at construction, other parameters are passed in as part of + method that require them + +commit c9422778224c14b136c15b2f82c035caa62ec47a +Author: David Turner +Date: Fri May 3 10:45:51 2019 +0200 + + Fix discovery config in docker-compose docs (#41753) + + Today's `docker-compose` docs are missing the `discovery.seed_nodes` config on + one of the nodes. With today's configuration the cluster can still form the + first time it is started, because `cluster.initial_master_nodes` requires both + nodes to bootstrap the cluster which ensures that each discover the other. + However if `es02` is elected master it will remove `es01` from the voting + configuration and then when restarted it will form a cluster on its own without + needing to do any discovery. Meanwhile `es01` doesn't know how to find `es02` + after a restart so will be unable to join this cluster. + + This commit fixes this by adding the missing configuration. + + Relates #41394, which fixes a different `docker-compose.yml` in the same way. + +commit 025619bbf115df738296da821ca4db6b76ad5909 +Author: Jack Conradson +Date: Thu May 2 11:28:30 2019 -0700 + + Improve error message for ln/log with negative results in function score + + This changes the error message for a negative result in a function score when + using the ln modifier to suggest using ln1p or ln2p when a negative result + occurs in a function score and for the log modifier to suggest using log1p or + log2p. + + This relates to #41509 + +commit 9f77ea26a9884bf452d5295e52ce545244a94395 +Author: lcawl +Date: Thu May 2 15:36:31 2019 -0700 + + [DOCS] Adds placeholder for 7.2 breaking changes + +commit d0f071236a62d8d102720740731364267cc1e8d5 +Author: Jason Tedor +Date: Thu May 2 16:35:55 2019 -0400 + + Simplify filtering addresses on interfaces (#41758) + + This commit is a refactoring of how we filter addresses on + interfaces. In particular, we refactor all of these methods into a + common private method. We also change the order of logic to first check + if an address matches our filter and then check if the interface is + up. This is to possibly avoid problems we are seeing where devices are + flapping up and down while we are checking for loopback addresses. We do + not expect the loopback device to flap up and down so by reversing the + logic here we avoid that problem on CI machines. Finally, we expand the + error message when this does occur so that we know which device is + flapping. + +commit 8421e3888700b2b14fc5b368400e21e1fed2d7cd +Author: Jay Modi +Date: Thu May 2 12:19:39 2019 -0600 + + Do not print null method name in reproduce line (#41691) + + This commit updates the reproduce line that is printed out when a test + fails so that it does not output `.null` as the method name when the + failure is not a specific method but a class level issue such as + threads being leaked from the SUITE. Previously, when this occurred the + reproduce line would look like: + + `./gradlew :server:integTest --tests "org.elasticsearch.indices.memory.breaker.CircuitBreakerServiceIT.null"` + + and after this change, the line no longer contains the `.null` after + the class name. + +commit 0d9797847a00b0defbfca6816047d6893a123d43 +Author: Hendrik Muhs +Date: Thu May 2 20:06:32 2019 +0200 + + remove validation methods in client (#41754) + + remove validation methods in client (#41754) + +commit 33b4032fab47fe41015316d40f54225f46c5e32c +Author: Benjamin Trent +Date: Thu May 2 12:01:59 2019 -0500 + + [ML] Correct indexer state on task re-allocation (#41724) (#41751) + +commit ab9154005b2073655e14cb7975eff96be39b6d9a +Author: Colin Goodheart-Smithe +Date: Thu May 2 09:14:00 2019 +0100 + + Adds version 6.7.3 + +commit a70f796eddc6ec8d5c3c778077c537e80a31f563 +Author: Benjamin Trent +Date: Thu May 2 11:09:42 2019 -0500 + + [ML] fix array oob in IDGenerator and adjust format for mapping (#41703) (#41717) + + * [ML] fix array oob in IDGenerator and adjust format for mapping + + * Update DataFramePivotRestIT.java + +commit 5d5370631002959cffc1c0c1d31d854ba944b79b +Author: Michael Basnight +Date: Thu May 2 10:39:45 2019 -0500 + + Add enrich policy DELETE API (#41495) + + This commit wires up the Rest calls and Transport calls for DELETE enrich + policy, as well as tests and rest spec additions. + +commit 2978ac3061f38bd0357fa4d9edb9ee2a0f45dd14 +Author: Michael Basnight +Date: Wed May 1 13:16:18 2019 -0500 + + Add enrich policy list API (#41553) + + This commit wires up the Rest calls and Transport calls for listing all + enrich policies, as well as tests and rest spec additions. + +commit b4bcbf9f64db54b7554c7471bb85dd9ff0c12ecd +Author: Tim Brooks +Date: Thu May 2 09:48:52 2019 -0600 + + Support http read timeouts for transport-nio (#41466) + + This is related to #27260. Currently there is a setting + http.read_timeout that allows users to define a read timeout for the + http transport. This commit implements support for this functionality + with the transport-nio plugin. The behavior here is that a repeating + task will be scheduled for the interval defined. If there have been + no requests received since the last run and there are no inflight + requests, the channel will be closed. + +commit a92c06ae099a432ee730dd3cb1d935e311b04817 +Author: Benjamin Trent +Date: Thu May 2 09:46:22 2019 -0500 + + [ML] Refactor NativeStorageProvider to enable reuse (#41414) (#41746) + + * [ML] Refactor NativeStorageProvider to enable reuse + + Moves `NativeStorageProvider` as a machine learning component + so that it can be reused for other job types. Also, we now + pass the persistent task description as unique identifier which + avoids conflicts between jobs of different type but with same ids. + + * Adding nativeStorageProvider as component + + Since `TransportForecastJobAction` is expected to get injected a `NativeStorageProvider` class, we need to make sure that it is a constructed component, as it does not have a zero parametered, public ctor. + +commit be7ec5a47a38f563bbc8fcdf54fbfd44039c2a8c +Author: Hendrik Muhs +Date: Thu May 2 16:06:31 2019 +0200 + + simplify indexer by moving members to base class (#41741) + + simplify indexer by moving members to base class + +commit 728fe2d4090179bea7cd543512632aa1ab68b84f +Author: Sandmannn +Date: Thu May 2 14:26:56 2019 +0200 + + Small correction in comments (#41623) + +commit bf1a70f242eb91a2f23837f139ffe11a7510d064 +Author: Christoph Büscher +Date: Thu May 2 11:49:26 2019 +0200 + + Mute RejectedExecutionTests#testHistoryAndTriggeredOnRejection + +commit e429cd7f286ba343e045870761063f05cf8b9900 +Merge: 8838bcc7761 b189596631b +Author: Martijn van Groningen +Date: Thu May 2 09:40:25 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit b189596631b8bf22c2d963b95c2dedf2d8a3a824 +Author: David Turner +Date: Thu May 2 08:27:45 2019 +0200 + + Add details to BulkShardRequest#getDescription() (#41711) + + Today a bulk shard request appears as follows in the detailed task list: + + requests[42], index[my_index] + + This change adds the shard index and refresh policy too: + + requests[42], index[my_index][2], refresh[IMMEDIATE] + +commit 81163455a8320922f8f183aca98564757fee9ae0 +Author: Toby McLaughlin +Date: Thu May 2 08:53:33 2019 +1000 + + Run packaging tests on RHEL 8 (#41662) + +commit 7c6d7997db052d75b993fe3a76c25af875f6f26a +Author: jaymode +Date: Wed May 1 15:12:05 2019 -0600 + + Fix skip version in indices open test + +commit b9e44288d3f16338f83af0293c69c185010492a4 +Author: Andy Bristol +Date: Wed May 1 13:16:00 2019 -0700 + + mute NodeTests#testCloseOnInterruptibleTask + + For #41448 + +commit 39b0b5809d17bd437afad33366bd991971125985 +Author: Jason Tedor +Date: Wed May 1 16:19:42 2019 -0400 + + Fix minimum compatible version after 6.8 + + This commit fixes the minimum compatible version after the introduction + of 6.8. + +commit 33945463f3bf85ba5e57ed0cdd78e32c94db1a0d +Author: James Rodewig +Date: Wed May 1 15:53:50 2019 -0400 + + [DOCS] Add `:` to render multiple inline macros in Asciidoctor (#41615) + +commit 7f7eb7b67933bcf665dd971931662f5a2b873eda +Author: Jay Modi +Date: Wed May 1 13:23:53 2019 -0600 + + Add version 7.0.2 to 7.x branch (#41715) + +commit 5fd3774d43baacec8aac3b0fcb811b07055790f0 +Author: lcawl +Date: Wed May 1 12:02:34 2019 -0700 + + [DOCS] Adds missing 7.2.0 highlights page + +commit 0870523489c0e59e33bb3bfe039b88ede1f0cd02 +Author: Jason Tedor +Date: Wed May 1 14:29:33 2019 -0400 + + Fix compilation in SecurityMocks + + This commit fixes compilation in SecurityMocks from what appears to be + some merge conflicts that were not resolved adequately. + +commit f500d727cfbef1014bb1c9441b7c486a325cab7f +Author: Jason Tedor +Date: Wed May 1 14:20:58 2019 -0400 + + Resolve conflicts in AuthenticationServiceTests + + This commit resolves some merge conflicts that arose in + AuthenticationServiceTests after a rebase. + +commit 942a1445f349c7e370221e3dd17729e21393282d +Author: Jason Tedor +Date: Wed May 1 13:52:05 2019 -0400 + + Fix reference to 7.1 in security token tests + + This version should be referencing 7.2 rather than 7.1, due to some + changes in timing of the token service changes. + +commit 842613055328548a4514a1128cff2889d61d9f81 +Author: Ioannis Kakavas +Date: Mon Apr 15 11:08:44 2019 +0300 + + Add negative tests for security features in basic + + Assert that API Keys, Tokens, DLS/FLS do not work in basic + +commit 3589ca84938ea15b1c318569c2332bfaec7e4ee9 +Author: Tim Vernum +Date: Thu Apr 11 17:13:03 2019 +1000 + + Add test for security on basic license. + + This is modelled on the qa test for TLS on basic. + + It starts a cluster on basic with security & performs a number of + security related checks. + It also performs those same checks on a trial license. + +commit 0ee16d0115703860e3ed24ff47c5b07ad1206587 +Author: Tim Vernum +Date: Thu Apr 11 10:30:56 2019 +1000 + + Security on Basic License + + This adds support for using security on a basic license. + It includes: + + - AllowedRealmType.NATIVE realms (reserved, native, file) + - Roles / RBAC + - TLS (already supported) + + It does not support: + + - Audit + - IP filters + - Token Service & API Keys + - Advanced realms (AD, LDAP, SAML, etc) + - Advanced roles (DLS, FLS) + - Pluggable security + + As with trial licences, security is disabled by default. + + This commit does not include any new automated tests, but existing tests have been updated. + +commit f08ac103eee9a180d7c75ed58cc7adfcf5728daa +Author: Jason Tedor +Date: Tue Apr 9 10:46:41 2019 -0400 + + Add 6.8 version constant + + This commit adds the 6.8 version constant to the 7.x branch. + +commit 7f3ab4524f8745b03b1e0025a56eb2e2dfe02b7a +Author: Jason Tedor +Date: Tue Apr 9 10:54:36 2019 -0400 + + Bump 7.x branch to version 7.2.0 + + This commit adds the 7.2.0 version constant to the 7.x branch, and bumps + BWC logic accordingly. + +commit c86f797df813ce54e6bab942efcc568e86ecd34a +Author: James Rodewig +Date: Wed May 1 09:28:12 2019 -0400 + + [DOCS] Reword `type` query deprecation note for Asciidoctor migration (#41595) + +commit c6abe74dd62280e004b701dd83f4111f7823131e +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed May 1 15:22:24 2019 +0200 + + Close and acquire commit during reset engine fix (#41584) (#41709) + + If closing a shard while resetting engine, + IndexEventListener.afterIndexShardClosed would be called while there is + still an active IndexWriter on the shard. For integration tests, this + leads to an exception during check index called from MockFSIndexStore + .Listener. Fixed. + + Relates to #38561 + +commit 990be1f806692981a0f8a8fed84b8cd3eed07e7e +Author: Albert Zaharovits +Date: Wed May 1 14:53:56 2019 +0300 + + Security Tokens moved to a new separate index (#40742) + + This commit introduces the `.security-tokens` and `.security-tokens-7` + alias-index pair. Because index snapshotting is at the index level granularity + (ie you cannot snapshot a subset of an index) snapshoting .`security` had + the undesirable effect of storing ephemeral security tokens. The changes + herein address this issue by moving tokens "seamlessly" (without user + intervention) to another index, so that a "Security Backup" (ie snapshot of + `.security`) would not be bloated by ephemeral data. + +commit bc333a5cbfcb2ae502843d524e75b4ee1bdc63e8 +Author: Benjamin Trent +Date: Wed May 1 06:44:29 2019 -0500 + + [ML] data frame, adding builder classes for complex config classes (#41638) (#41704) + + * [ML] data frame, adding builder classes for complex config classes + + * Addressing PR comments, adding some java docs + + * cleaning up constructor + + * fixing indentation + + * change constructors to be package-private + +commit 26c72c96bdb248b595809a3307f4014c3d1b6a98 +Author: Jason Tedor +Date: Wed May 1 07:21:16 2019 -0400 + + Fix imports in KeyStoreWrapperTests + + This commit addresses a checkstyle violation in KeyStoreWrapperTests, + removing a leftover import. + +commit 0b46a62f6be031b47d99203d69d15eb5e454db03 +Author: Jason Tedor +Date: Wed May 1 06:46:31 2019 -0400 + + Drop distinction in entries for keystore (#41701) + + Today we allow adding entries from a file or from a string, yet we + internally maintain this distinction such that if you try to add a value + from a file for a setting that expects a string or add a value from a + string for a setting that expects a file, you will have a bad time. This + causes a pain for operators such that for each setting they need to know + this difference. Yet, we do not need to maintain this distinction + internally as they are bytes after all. This commit removes that + distinction and includes logic to upgrade legacy keystores. + +commit 7ac328350918bb004108ad982641672d5ede8004 +Author: Jake Landis +Date: Tue Apr 30 16:13:24 2019 -0500 + + Fix Watcher deadlock that can cause in-abilty to index documents. (#41418) (#41684) + + This commit removes the usage of the `BulkProcessor` to write history documents + and delete triggered watches on a `EsRejectedExecutionException`. Since the + exception could be handled on the write thread, the write thread can be blocked + waiting on watcher threads (due to a synchronous method). This is problematic + since those watcher threads can be blocked waiting on write threads. + + This commit also moves the handling of the exception to the generic threadpool + to avoid submitting write requests from the write thread pool. + + fixes #41390 + +commit b3f4533e1cce3d25e89c8b1b1e3f99529badcfea +Author: Tom Veasey +Date: Tue Apr 30 21:47:54 2019 +0100 + + [ML] Update for model selection change and disable temporarily (#41482) (#41682) + +commit 66d8549083d90ab93cd1b04586fa1bcbc8d660ab +Author: James Rodewig +Date: Tue Apr 30 16:40:26 2019 -0400 + + [DOCS] Move block delimiter and anchor in 'Configuring Security' for Asciidoctor migration (#41604) + +commit b65ceb36bceb2bf1ecad6bed27d350f90d6e08c5 +Author: James Rodewig +Date: Tue Apr 30 16:30:58 2019 -0400 + + [DOCS] Escape quotes to avoid smart quotes in Asciidoctor (#41603) + +commit 721787fa76e553dd29717085bf891124ef314fe5 +Author: James Rodewig +Date: Tue Apr 30 16:22:15 2019 -0400 + + [DOCS] Remove soft line breaks (+) for Asciidoctor migration (#41600) + +commit 3e681648258abe8ef1b6a307e6d335c796a6895e +Author: James Rodewig +Date: Tue Apr 30 16:02:59 2019 -0400 + + [DOCS] Move beta[] tag for Asciidoctor migration (#41599) + +commit 9506e3f1c5ce4fbab06e2a23fd751ab73435fc60 +Author: James Rodewig +Date: Tue Apr 30 15:50:52 2019 -0400 + + [DOCS] Escape commas in deprecated[] for Asciidoctor migration (#41598) + +commit d329186972ebbf2e9c66f79385abff6541afd1be +Author: James Rodewig +Date: Tue Apr 30 15:18:49 2019 -0400 + + [DOCS] Escape depcreated[] note for Asciidoctor migration (#41608) + +commit 887f3f2c833a82529edebd9b59f505cbb6b2bce7 +Author: Nhat Nguyen +Date: Mon Apr 29 20:57:31 2019 -0400 + + Simplify initialization of max_seq_no of updates (#41161) + + Today we choose to initialize max_seq_no_of_updates on primaries only so + we can deal with a situation where a primary is on an old node (before + 6.5) which does not have MUS while replicas on new nodes (6.5+). + However, this strategy is quite complex and can lead to bugs (for + example #40249) since we have to assign a correct value (not too low) to + MSU in all possible situations (before recovering from translog, + restoring history on promotion, and handing off relocation). + + Fortunately, we don't have to deal with this BWC in 7.0+ since all nodes + in the cluster should have MSU. This change simplifies the + initialization of MSU by always assigning it a correct value in the + constructor of Engine regardless of whether it's a replica or primary. + + Relates #33842 + +commit 8838bcc776109c21713b1021d9d0b871a61f506d +Author: Martijn van Groningen +Date: Tue Apr 30 19:42:19 2019 +0200 + + Add enrich processor (#41532) + + The enrich processor performs a lookup in a locally allocated + enrich index shard using a field value from the document being enriched. + If there is a match then the _source of the enrich document is fetched. + The document being enriched then gets the decorate values from the + enrich document based on the configured decorate fields in the pipeline. + + Note that the usage of the _source field is temporary until the enrich + source field that is part of #41521 is merged into the enrich branch. + Using the _source field involves significant decompression which not + desired for enrich use cases. + + The policy contains the information what field in the enrich index + to query and what fields are available to decorate a document being + enriched with. + + The enrich processor has the following configuration options: + * `policy_name` - the name of the policy this processor should use + * `enrich_key` - the field in the document being enriched that holds to lookup value + * `ignore_missing` - Whether to allow the key field to be missing + * `enrich_values` - a list of fields to decorate the document being enriched with. + Each entry holds a source field and a target field. + The source field indicates what decorate field to use that is available in the policy. + The target field controls the field name to use in the document being enriched. + The source and target fields can be the same. + + Example pipeline config: + + ``` + { + "processors": [ + { + "policy_name": "my_policy", + "enrich_key": "host_name", + "enrich_values": [ + { + "source": "globalRank", + "target": "global_rank" + } + ] + } + ] + } + ``` + + In the above example documents are being enriched with a global rank value. + For each document that has match in the enrich index based on its host_name field, + the document gets an global rank field value, which is fetched from the `globalRank` + field in the enrich index and saved as `global_rank` in the document being enriched. + + This is PR is part one of #41521 + +commit d46f55f013a5576f2bc21cd7c3877d930796b8b2 +Author: James Rodewig +Date: Tue Apr 30 14:11:24 2019 -0400 + + [DOCS] Add attribute to escape minimal pt token link in Asciidoctor (#41613) + +commit 005296dac6ad2ed7f5366558247c615664aa488b +Author: James Rodewig +Date: Tue Apr 30 13:46:13 2019 -0400 + + [DOCS] Allow attribute substitution in titleabbrevs for Asciidoctor migration (#41574) + + * [DOCS] Replace attributes in titleabbrevs for Asciidoctor migration + + * [DOCS] Add [subs="attributes"] so attributes render in Asciidoctor + + * Revert "[DOCS] Replace attributes in titleabbrevs for Asciidoctor migration" + + This reverts commit 98f130257a7c71e9f6cddf5157af7886418338d8. + + * [DOCS] Fix merge conflict + +commit 7cc4b9a8b37085622ab646ec8d2147828a8ebbd4 +Author: Armin Braun +Date: Tue Apr 30 17:03:57 2019 +0200 + + Implement Bulk Deletes for GCS Repository (#41368) (#41681) + + * Implement Bulk Deletes for GCS Repository (#41368) + + * Just like #40322 for AWS + * We already had a bulk delete API but weren't using it from the blob container implementation, now we are using it + * Made the bulk delete API also compliant with our interface that only suppresses errors about non existent blobs by stating failed deletes (I didn't use any bulk stat action here since having to stat here should be the exception anyway and it would make error handling a lot more complex) + * Fixed bulk delete API to limit its batch size to 100 in line with GCS recommendations + + back port of #41368 + +commit 53702efddd67f3bb95d3a566cacb747d8bc73c58 +Author: James Rodewig +Date: Tue Apr 30 10:19:09 2019 -0400 + + [DOCS] Add anchors for Asciidoctor migration (#41648) + +commit c26b8eb4de522000444b57446572ac4d3c4f8a04 +Author: James Rodewig +Date: Tue Apr 30 09:58:09 2019 -0400 + + [DOCS] Remove line break from deprecated[] macro (#41616) + +commit cc39233ff8404aaf60f00f0538ef916e021abb98 +Author: James Rodewig +Date: Tue Apr 30 09:25:04 2019 -0400 + + [DOCS] Add soft line breaks to maintain nested definition list (#41617) + +commit f737b05ad177deb075e62a1a3c37187f240f6ff2 +Author: David Kyle +Date: Tue Apr 30 09:45:35 2019 +0100 + + Mute CcrRetentionLeaseIT.testForgetFollower + + https://github.com/elastic/elasticsearch/issues/39850 + +commit 08c0ecb90e77afb6234ee09853ee52681b6875cb +Author: Armin Braun +Date: Tue Apr 30 09:27:51 2019 +0200 + + Upgrade to Netty 4.1.35 (#41499) (#41651) + + * Some fixes and possible performance fixes in the last 3 versions -> + upgrading + +commit df3ef662942d9074b7d48661bfffc4ff3505660a +Author: Tim Brooks +Date: Mon Apr 29 17:59:13 2019 -0600 + + Remove dedicated SSL network write buffer (#41654) + + This is related to #27260. Currently for the SSLDriver we allocate a + dedicated network write buffer and encrypt the data into that buffer one + buffer at a time. This requires constantly switching between encrypting + and flushing. This commit adds a dedicated outbound buffer for SSL + operations that will internally allocate new packet sized buffers as + they are need (for writing encrypted data). This allows us to totally + encrypt an operation before writing it to the network. Eventually it can + be hooked up to buffer recycling. + + This commit also backports the following commit: + + Handle WRAP ops during SSL read + + It is possible that a WRAP operation can occur while decrypting + handshake data in TLS 1.3. The SSLDriver does not currently handle this + well as it does not have access to the outbound buffer during read call. + This commit moves the buffer into the Driver to fix this issue. Data + wrapped during a read call will be queued for writing after the read + call is complete. + +commit 10ab838106d3c5678235be60f2f9255d1986711b +Author: Igor Motov +Date: Mon Apr 29 19:43:31 2019 -0400 + + Geo: Add GeoJson parser to libs/geo classes (#41575) (#41657) + + Adds GeoJson parser for Geometry classes defined in libs/geo. + + Relates #40908 and #29872 + +commit 92a820bc1a1e07c5c83e15b48ebaeeda1044d277 +Author: Benjamin Trent +Date: Mon Apr 29 10:14:17 2019 -0500 + + [ML] Add bucket_script agg support to data frames (#41594) (#41639) + +commit a01f451ef707c16d2b99f84290a94095493e6c8d +Author: Alan Woodward +Date: Mon Apr 29 13:19:56 2019 +0100 + + Limit complexity of IntervalQueryBuilderTests#testRandomSource() (#41538) + + IntervalsSources can throw IllegalArgumentExceptions if they would produce + too many disjunctions. To mitigate against this when building random + sources, we limit the depth of the randomly generated source to four + nested sources + + Fixes #41402 + +commit b23709b17820bd452ad73e8eb20297188b5411b0 +Author: Dan Hermann +Date: Mon Apr 29 07:31:01 2019 -0500 + + Applies the same naming restrictions to repositories as to snapshots except that leading underscores and uppercase characters are permitted. (#41585) + + Fixes #40817. + +commit fc4f401214d79faa85a796a0f1b4ae5d8aee8f73 +Author: James Rodewig +Date: Mon Apr 29 08:28:03 2019 -0400 + + [DOCS] Document 200 http code response for partial results (#40367) + +commit 6e51b6f96d33827e9565421a6cf3e4af0c70b6b7 +Author: Armin Braun +Date: Mon Apr 29 12:01:58 2019 +0200 + + Add Repository Consistency Assertion to SnapshotResiliencyTests (#41631) + + * Add Repository Consistency Assertion to SnapshotResiliencyTests (#40857) + + * Add Repository Consistency Assertion to SnapshotResiliencyTests + + * Add some quick validation on not leaving behind any dangling metadata or dangling indices to the snapshot resiliency tests + * Added todo about expanding this assertion further + + * Fix SnapshotResiliencyTest Repo Consistency Check (#41332) + + * Fix SnapshotResiliencyTest Repo Consistency Check + + * Due to the random creation of an empty `extra0` file by the Lucene mockFS we see broken tests because we use the existence of an index folder in assertions and the index deletion doesn't go through if there are extra files in an index folder + * Fixed by removing the `extra0` file and resulting empty directory trees before asserting repo consistency + * Closes #41326 + + * Reenable SnapshotResiliency Test (#41437) + + This was fixed in https://github.com/elastic/elasticsearch/pull/41332 + but I forgot to reenable the test. + + * fix compile on java8 + +commit 57adee0c630323fd5c83da5ed1db5232ca6d90e6 +Author: Martijn van Groningen +Date: Mon Apr 29 10:31:30 2019 +0200 + + fixed compile error after merging in the 7.x branch + +commit eb9618f1b7dac620acf5e711f199cc1cd63631fa +Merge: 6af17e4bdfd 1a6ffb26444 +Author: Martijn van Groningen +Date: Mon Apr 29 09:21:04 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 1a6ffb2644449bcc32bd14e7fd9e1f85b2d1f1f8 +Author: David Kyle +Date: Mon Apr 8 14:18:40 2019 +0100 + + Mute ClusterPrivilegeTests.testThatSnapshotAndRestore + + Tracked in #38030 + +commit 719e4452afed733bdf52362dce527b33212a8349 +Author: Yogesh Gaikwad +Date: Mon Apr 29 13:56:31 2019 +1000 + + Revert "Suppress illegal access in plugin install (#41620)" + + This reverts commit fb9f729426c87c9ebc400c3207129f276947f16e. + +commit fb9f729426c87c9ebc400c3207129f276947f16e +Author: Jason Tedor +Date: Sun Apr 28 23:03:34 2019 -0400 + + Suppress illegal access in plugin install (#41620) + + We use Bouncy Castle to verify signatures when installing official + plugins. This leads to illegal access warnings because Bouncy Castle + accesses the Sun security provider constructor. This commit adds an + add-opens flag to suppress this illegal access. + +commit 3b3f7b7e3458f7714aba15d91198aa112a624c31 +Author: Jason Tedor +Date: Sun Apr 28 21:04:08 2019 -0400 + + Bump the bundled JDK to 12.0.1 (#41627) + + This commit bumps the bundled JDK to version 12.0.1. Note that we had to + add a new pattern here as Oracle has changed the source of the + builds. This commit will be backported to 6.7 in a different form to + bump the bundled JDK in the Docker images too. + +commit 7a82277bc3e93f299ac2305325f1695f0435467f +Author: Jason Tedor +Date: Sun Apr 28 20:14:17 2019 -0400 + + Change JDK distribution source (#41626) + + We had been obtaining JDK distributions from download.java.net. This + site is now presenting a certificate that does not list + download.java.net as a SAN. Therefore with host verification, the build + can not use this site. This commit switches to using download.oracle.com + which appears to be an alternative name for the same CNAME + download.oracle.com.edgekey.net. This allows our builds to resume. + +commit c0d40ae4caaf712f83812c32eeadcc8adab82569 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sun Apr 28 21:25:42 2019 +1000 + + Remove deprecated stashWithOrigin calls and use the alternative (#40847) (#41562) + + This commit removes the deprecated `stashWithOrigin` and + modifies its usage to use the alternative. + +commit a8c3c5cd78a25b7af884c3d6867958eb78f23842 +Author: Jim Ferenczi +Date: Sat Apr 27 18:37:15 2019 +0200 + + Restore rolling upgrade test for the _all field (#41462) + + This commit adapts the rolling upgrade test introduced in #37808 + to ignore runs that upgrade from a version on or after 7.0. + + Closes #41453 + +commit 615a0211f0a0ba681aa4acadb4fdb3027128960b +Author: Nhat Nguyen +Date: Tue Apr 23 21:44:25 2019 -0400 + + Recovery should not indefinitely retry on mapping error (#41099) + + A stuck peer recovery in #40913 reveals that we indefinitely retry on + new cluster states if indexing translog operations hits a mapper + exception. We should not wait and retry if the mapping on the target is + as recent as the mapping that the primary used to index the replaying + operations. + + Relates #40913 + +commit 75283294f587bf2fcd46858ed6e004380c4b4864 +Author: Michael Morello +Date: Sat Apr 27 12:18:04 2019 +0200 + + Fix multi-node parsing in voting config exclusions REST API (#41588) + + Fixes an issue where multiple nodes where not properly parsed in the voting config exclusions REST API. + + Closes #41587 + +commit a0990ca239758a62f45a4eaa71bfbf8bd80f9971 +Author: Benjamin Trent +Date: Fri Apr 26 16:50:59 2019 -0500 + + [ML] cleanup + adding description field to transforms (#41554) (#41605) + + * [ML] cleanup + adding description field to transforms + + * making description length have a max of 1k + +commit 858e7f4a620c0286230365dfe67c9eb25e21568c +Author: Chris Earle +Date: Fri Apr 26 16:44:03 2019 -0400 + + [7.x] [Monitoring] Add `usage` mapping for `monitoring-kibana` index (#40899) (#41601) + + Backports the usage change to 7.x. A separate backport is needed to + change the version for 6.7, which will complete this backport. + +commit 22f3b53ed706476695cc990d2e5638e153a4ce0b +Author: Hicham Mallah +Date: Fri Apr 26 22:00:11 2019 +0300 + + Deprecate using 0 value for `min_children` in `has_child` query (#41555) + + After changing the allowed minimum value for min_children in has_child query from 0 to 1 in + the next major version, this PR adds a deprecation warning for these cases. + + Closes #41548 + +commit 6af17e4bdfdaff9f509111dbeba618d600311e7a +Author: Martijn van Groningen +Date: Fri Apr 26 20:19:19 2019 +0200 + + Add enrich qa module for rest tests and (#41568) + + move put policy api yaml test to this rest module. + + The main benefit is that all tests will then be run when running: + `./gradlew -p x-pack/plugin/enrich check` + + The rest qa module starts a node with default distribution and basic + license. + + This qa module will also be used for adding different rest tests (not yaml), + for example rest tests needed for #41532 + + Also when we are going to work on security integration then we can + add a security qa module under the qa folder. Also at some point + we should add a multi node qa module. + +commit d7ab86db9c1d8e1a4c5ece3b120b076cfc8b1eb6 +Author: Alan Woodward +Date: Thu Apr 4 09:07:42 2019 +0100 + + Document restrictions on fuzzy matching when using synonyms (#40783) + + Relates to #25518 #41592 + +commit d54890185543aa8bee3d002ac3df4760b5f32107 +Author: James Rodewig +Date: Fri Apr 26 12:12:57 2019 -0400 + + [DOCS] Add space to fix Asciidoctor output (#41579) + +commit 335f2bf102211b7f0b82bae85f8318a42ed59997 +Author: Alpar Torok +Date: Fri Apr 26 18:34:03 2019 +0300 + + Testclsuters: convert plugins qa projects (#41496) + + Add testclusters support for files in keystore and convert qa subprojects within plugins. + +commit 06827a495fb8681b7b75ff182f6a5f31a4077fc7 +Author: James Rodewig +Date: Fri Apr 26 11:41:19 2019 -0400 + + [DOCS] Fix delimited blocks for Asciidoctor migration (#41583) + +commit 4adf7963c7ce79c2dc8fa027d98ec8e5b20dd725 +Author: James Rodewig +Date: Fri Apr 26 11:35:07 2019 -0400 + + [DOCS] Escape commas in experimental[] for Asciidoctor migration (#41578) + +commit 3ccb48e516b1b61cf7dcf83037543bed84bad71d +Author: Benjamin Trent +Date: Fri Apr 26 10:23:43 2019 -0500 + + [ML] data frame, verify primary shards are active for configs index before task start (#41551) (#41580) + +commit f8081e855815603ced116e387a41fc1c0693eac0 +Author: James Rodewig +Date: Fri Apr 26 11:20:34 2019 -0400 + + [DOCS] Fix ID for Asciidoctor migration (#41577) + +commit 113b24be4b8a5725468e322b261606f0fe5a452b +Author: Nick Knize +Date: Thu Apr 25 11:59:13 2019 -0500 + + Refactor GeoHashUtils (#40869) + + This commit refactors GeoHashUtils class into a new Geohash utility class located in the ES geo library. The intent is to not only better control what geo methods are whitelisted for painless scripting but to clean up the geo utility API in general. + +commit 6a7459ff112c82262b7b2243c2dc2b774464712a +Author: James Rodewig +Date: Fri Apr 26 10:23:30 2019 -0400 + + [DOCS] Clarify Recovery Settings for Shard Relocation (#40329) + + * Clarify that peer recovery settings apply to shard relocation + + * Fix awkward wording of 1st sentence + + * [DOCS] Remove snapshot recovery reference. + Call out link to [[cat-recovery]]. + Separate expert settings. + +commit aad33121d8ac0162b926a66e30eec517566d5d8d +Author: Armin Braun +Date: Fri Apr 26 15:36:09 2019 +0200 + + Async Snapshot Repository Deletes (#40144) (#41571) + + Motivated by slow snapshot deletes reported in e.g. #39656 and the fact that these likely are a contributing factor to repositories accumulating stale files over time when deletes fail to finish in time and are interrupted before they can complete. + + * Makes snapshot deletion async and parallelizes some steps of the delete process that can be safely run concurrently via the snapshot thread poll + * I did not take the biggest potential speedup step here and parallelize the shard file deletion because that's probably better handled by moving to bulk deletes where possible (and can still be parallelized via the snapshot pool where it isn't). Also, I wanted to keep the size of the PR manageable. + * See https://github.com/elastic/elasticsearch/pull/39656#issuecomment-470492106 + * Also, as a side effect this gives the `SnapshotResiliencyTests` a little more coverage for master failover scenarios (since parallel access to a blob store repository during deletes is now possible since a delete isn't a single task anymore). + * By adding a `ThreadPool` reference to the repository this also lays the groundwork to parallelizing shard snapshot uploads to improve the situation reported in #39657 + +commit 4127d6889b98124e8fa16a2f50f44b831ec8cba4 +Author: Mayya Sharipova +Date: Fri Apr 26 08:50:27 2019 -0400 + + Remove experimental label froms script_score query (#41572) + +commit 4836ff7bcda20d1f3c76176fd0e13d8886eeb6ee +Author: Benjamin Trent +Date: Fri Apr 26 07:18:49 2019 -0500 + + [ML] add multi node integ tests for data frames (#41508) (#41552) + + * [ML] adding native-multi-node-integTests for data frames' + + * addressing streaming issues + + * formatting fixes + + * Addressing PR comments + +commit 1f00cec36ff5905ec5416255697fceefd21f886b +Author: David Kyle +Date: Fri Apr 26 12:01:21 2019 +0100 + + [Ml-Dataframe] Update URLs in Data frame client java doc (#41539) + +commit 7824f60a34da984a838cb61f6aab12c9148ef948 +Author: Armin Braun +Date: Fri Apr 26 10:59:09 2019 +0200 + + Simplify Snapshot Resiliency Test (#40930) (#41565) + + * Thanks to #39793 dynamic mapping updates don't contain blocking operations anymore so we don't have to manually put the mapping in this test and can keep it a little simpler + +commit 67820f9da1929fde373fa4f0b59fb4d998fc5d84 +Author: Jim Ferenczi +Date: Fri Apr 26 10:18:48 2019 +0200 + + Fix search_as_you_type's sub-fields to pick their names from the full path of the root field (#41541) + + The subfields of the search_as_you_type are prefixed with the name of their root field. + However they should used the full path of the root field rather than just the name since + these fields can appear in a multi-`fields` definition or under an object field. + Since this field type is not released yet, this should be considered as a non-issue. + +commit 078936b8f573b10be990c9db9fd60b0b7520af8e +Author: Christoph Büscher +Date: Fri Apr 26 09:48:11 2019 +0200 + + Remove search analyzers from DocumentFieldMappers (#41484) + + These references seem to be unused except for tests and should be removed to + keep the places we store analyzers limited. + +commit 6a24fd3f260e03fd022e65bac6d6710850c68abd +Author: Armin Braun +Date: Fri Apr 26 09:04:34 2019 +0200 + + Add Restore Operation to SnapshotResiliencyTests (#40634) (#41546) + + * Add Restore Operation to SnapshotResiliencyTests + + * Expand the successful snapshot test case to also include restoring the snapshop + * Add indexing of documents as well to be able to meaningfully verify the restore + * This is part of the larger effort to test eventually consistent blob stores in #39504 + +commit 6996739a093d6482b114b94895dfac284b68eb6e +Author: Alexander Reelsen +Date: Fri Apr 26 08:56:34 2019 +0200 + + Update community client and integration docs (#41513) + + Added integrations for a couple of frameworks. + + Removed community clients where the last commit was more than three + years ago. Also added the official go client link and removed the + official groovy client, as it is outdated. + +commit 13fa72cae32788f86d52cd3b976ebbd6a9c5a892 +Author: Tim Vernum +Date: Fri Apr 26 15:43:48 2019 +1000 + + Fix broken test on FIPS for specific seed (#41230) + + Under random seed 4304ED44CB755610 the generated byte pattern causes + BC-FIPS to throw + + java.io.IOException: DER length more than 4 bytes: 101 + + Rather than simply returning an empty list (as it does for most random + values). + + Backport of: #40939 + +commit 1f8ff052a14205ed0014ea70ae2c6cccd1d978f8 +Author: Tim Brooks +Date: Thu Apr 25 18:39:25 2019 -0600 + + Revert "Remove dedicated SSL network write buffer (#41283)" + + This reverts commit f65a86c258de78410365b9fdae873d278c017723. + +commit c4cb0507b4bd2dc78197f283347efc6754de0c17 +Author: Nhat Nguyen +Date: Thu Apr 25 17:50:29 2019 -0400 + + Mute testDriverConfigurationWithSSLInURL + + Tracked at #41557 + +commit b288b88ba09bdfc870902d44eae3e465ab688799 +Author: Costin Leau +Date: Mon Apr 22 22:41:45 2019 +0300 + + SQL: Use field caps inside DESCRIBE TABLE as well (#41377) + + Thanks to #34071, there is enough information in field caps to infer + the table structure and thus use the same API consistently across the + IndexResolver. + + (cherry picked from commit f99946943a3350206b6bca774b2f060f41a787b3) + +commit f65a86c258de78410365b9fdae873d278c017723 +Author: Tim Brooks +Date: Thu Apr 25 14:30:16 2019 -0600 + + Remove dedicated SSL network write buffer (#41283) + + This is related to #27260. Currently for the SSLDriver we allocate a + dedicated network write buffer and encrypt the data into that buffer one + buffer at a time. This requires constantly switching between encrypting + and flushing. This commit adds a dedicated outbound buffer for SSL + operations that will internally allocate new packet sized buffers as + they are need (for writing encrypted data). This allows us to totally + encrypt an operation before writing it to the network. Eventually it can + be hooked up to buffer recycling. + +commit fad45ea6bdd3d77e41dc290d2af4fe380b341178 +Author: Michael Basnight +Date: Thu Apr 25 15:13:59 2019 -0500 + + Add enrich policy PUT API (#41383) + + This commit wires up the Rest calls and Transport calls for PUT enrich + policy, as well as tests and rest spec additions. + +commit 08c5d3b91235b6d8abf34d9da7266797436bcdd1 +Author: James Rodewig +Date: Thu Apr 25 15:06:35 2019 -0400 + + [DOCS] Explicitly set section IDs for Asciidoctor migration (#41547) + + * [DOCS] Explicitly set section ID for faster phrase queries + + * [DOCS] Explicitly set section ID for faster prefix queries + +commit 52495843ccd255072728a011a4f8fde193fe6388 +Author: Christoph Büscher +Date: Thu Apr 25 20:47:03 2019 +0200 + + [Docs] Fix common word repetitions (#39703) + +commit 6d7110edf5e02c115c73e49a61c2ddcf36dd5009 +Author: Tim Brooks +Date: Tue Apr 23 19:43:15 2019 -0600 + + SSLDriver can transition to CLOSED in handshake (#41458) + + TLS 1.3 changes to the SSLEngine introduced a scenario where a UNWRAP + call during a handshake can consume a close notify alerty without + throwing an exception. This means that we continue down a codepath where + we assert that we are still in handshaking mode. Transitioning to closed + from handshaking is a valid scenario. This commit removes this + assertion. + +commit 2e255a64462d6d2497a98ad5bb7b9aeb2d0b3fdf +Author: James Rodewig +Date: Thu Apr 25 13:58:48 2019 -0400 + + [DOCS] Note TESTRESPONSE can't be used immediately after TESTSETUP (#41542) + +commit a02f5544febcee2966e403d0eec56f24f9bfacca +Author: RomainGeffraye +Date: Thu Apr 25 19:28:36 2019 +0200 + + Update configuring-ldap-realm.asciidoc (#40427) + +commit 20bdd42a2dc1fd33d31ab5bf48c2ea19e5fd8ec5 +Author: Jash Gala +Date: Thu Apr 25 22:34:27 2019 +0530 + + Fixed very small typo in date (#41398) + +commit 23b3741618b016623cdc6a92afbf825d1bcbaef9 +Author: Armin Braun +Date: Thu Apr 25 18:25:03 2019 +0200 + + Remove Exists Check from S3 Repository Deletes (#40931) (#41534) + + * The check doesn't add much if anything practically, since the S3 repository is eventually consistent and we only log the non-existence of a blob anyway + * We don't do the check on writes for this very reason and documented it as such + * Removing the check saves one API call per single delete speeding up the deletion process and lowering costs + +commit 08843ba62b9401952a073ad65683a59592207c60 +Author: Benjamin Trent +Date: Thu Apr 25 11:23:12 2019 -0500 + + [ML] Adds progress reporting for transforms (#41278) (#41529) + + * [ML] Adds progress reporting for transforms + + * fixing after master merge + + * Addressing PR comments + + * removing unused imports + + * Adjusting afterKey handling and percentage to be 100* + + * Making sure it is a linked hashmap for serialization + + * removing unused import + + * addressing PR comments + + * removing unused import + + * simplifying code, only storing total docs and decrementing + + * adjusting for rewrite + + * removing initial progress gathering from executor + +commit 6184efaff6e74b37c6ec9437516eca0b06edcc31 +Author: Jim Ferenczi +Date: Thu Apr 25 18:13:48 2019 +0200 + + Handle unmapped fields in _field_caps API (#34071) (#41426) + + Today the `_field_caps` API returns the list of indices where a field + is present only if this field has different types within the requested indices. + However if the request is an index pattern (or an alias, or both...) there + is no way to infer the indices if the response contains only fields that have + the same type in all indices. This commit changes the response to always return + the list of indices in the response. It also adds a way to retrieve unmapped field + in a specific section per field called `unmapped`. This section is created for each field + that is present in some indices but not all if the parameter `include_unmapped` is set to + true in the request (defaults to false). + +commit 40aef2b8aa2a78e6ffcfd795ce073bf4ba3aba1f +Author: Armin Braun +Date: Thu Apr 25 16:05:04 2019 +0200 + + Introduce Delegating ActionListener Wrappers (#40129) (#41527) + + * Introduce Delegating ActionListener Wrappers + * Dry up use cases of ActionListener that simply pass through the response or exception to another listener + +commit d119abdf9643a7ef953a63ef3069a17e099bfb5e +Author: Ignacio Vera +Date: Thu Apr 25 15:25:48 2019 +0200 + + Improve accuracy for Geo Centroid Aggregation (#41514) + + keeps the partial results as doubles and uses Kahan summation to help reduce floating point errors. + +commit cd830b53e27e5049e66e9316f79d229fe85d8c9d +Author: Armin Braun +Date: Thu Apr 25 13:18:03 2019 +0200 + + Name Snapshot Data Blobs by UUID (#40652) (#41523) + + * Name Snapshot Data Blobs by UUID + + * There is no functional reason why we need incremental naming for these files but + * As explained in #38941 it is a possible source of corrupting the repository + * It wastes API calls for the list operation + * Is just needless complication + * Since we store the exact names of the data blobs in all the metadata anyway, we can make this change without any BwC considerations + * Even on the worst case scenario of a downgrade the functionality would continue working since the incremental names wouldn't conflict with the uuids and the number parsing for finding the next incremental name suppresses the exception when encountring a non-numeric value after the double underscore prefix + +commit 31d5a5aa36ffe5a4fd6cf25a1bcd7c7a106868c3 +Author: David Kyle +Date: Thu Apr 25 12:08:14 2019 +0100 + + Revert "Clean up clusters between tests (#41187)" + + This reverts commit 44a0c468cf7da451ee572ccdfb3913283efbe16f. + +commit 8a0e5f7b87a4930f0077478350a0ab9b2c56a4f1 +Author: Luca Cavanna +Date: Thu Apr 25 12:45:18 2019 +0200 + + Deprecate support for first line empty in msearch API (#41442) + + In order to support empty action metadata in the first msearch item, + we need to remove support for prepending msearch request body with an + empty line, which prevents us from parsing the empty line as action + metadata for the first search item. + + Relates to #41011 + +commit 906f88029b3b001dc6d0f9e899f2e33055a694ec +Author: Przemyslaw Gomulka +Date: Thu Apr 25 12:21:01 2019 +0200 + + Remove the test which is testing java and joda api backport(#41493) #41518 + + The test is testing the java time API and fails in case it hits daylight saving time changes. + Java time has the right implementation and we don't need to test this. + more details on how the test was affected by the DST change on this comment + closes #39617 + backport(#41493) + +commit fe5789ada1d2cef8ce569a5847c4ce0573584558 +Author: Albert Zaharovits +Date: Thu Apr 25 12:02:09 2019 +0300 + + Fix Has Privilege API check on restricted indices (#41226) + + The Has Privileges API allows to tap into the authorization process, to validate + privileges without actually running the operations to be authorized. This commit + fixes a bug, in which the Has Privilege API returned spurious results when checking + for index privileges over restricted indices (currently .security, .security-6, + .security-7). The actual authorization process is not affected by the bug. + +commit 7e3875d78118774f9ea705a3993aec500f059a3c +Author: Ryan Ernst +Date: Wed Apr 24 23:39:40 2019 -0700 + + Upgrade hamcrest to 2.1 (#41464) + + hamcrest has some improvements in newer versions, like FileMatchers + that make assertions regarding file exists cleaner. This commit upgrades + to the latest version of hamcrest so we can start using new and improved + matchers. + +commit 257682918f3e198a32b4d7804786058aa04bbeac +Author: Alexander Reelsen +Date: Thu Apr 25 05:36:52 2019 +0200 + + Remove stale integrations in docs (#40764) + + Removed all integrations in documentation, who have not seen a + commit for more than three years when the website + returned an error. + +commit a61ec11f36c083dd5beac83d3cb4025d53080e28 +Author: Martijn van Groningen +Date: Wed Apr 24 20:28:50 2019 +0200 + + Expose Engine.Searcher provider to ingest plugins. (#41010) + + Relates to #32789 + +commit 7c819fd2aa7cc591c133e1f415f70e687a98ba09 +Author: Armin Braun +Date: Wed Apr 24 20:46:21 2019 +0200 + + Fix BulkRejectionIT (#41446) (#41500) + + * Due to #40866 one of the two parallel bulk requests can randomly be + rejected outright when the write queue is full already, we can catch + this situation and ignore it since we can still have the rejection for + the dynamic mapping udate for the other reuqest and it's somewhat rare + to run into this anyway + * Closes #41363 + +commit bb6ca25f700e0e4acd8fa44913d80f485f6254f8 +Author: Nik Everett +Date: Wed Apr 24 14:44:11 2019 -0400 + + Docs: Specify an ID in migration docs (#41501) + + We link to these migraiton docs but we don't specify the id. This + isn't great practice in general and is preventing us from migrating to + Asciidoctor because it generates ids in a slightly different way. + +commit ec5dd0594f4153f78a6c220519a952973acf517b +Author: Zachary Tong +Date: Wed Apr 24 13:22:06 2019 -0400 + + Disallow null/empty or duplicate composite sources (#41359) + + Adds some validation to prevent duplicate source names from being + used in the composite agg. + + Also refactored to use a ConstructingObjectParser and removed the + private ctor and setter for sources, making it mandatory. + +commit 1db9166ea07f1400ef970655ae45b239743dae5a +Author: Armin Braun +Date: Wed Apr 24 18:39:17 2019 +0200 + + Fix Broken Index Shard Snapshot File Preventing Snapshot Creation (#41310) (#41473) + + * The problem here is that if we run into a corrupted index-N file, instead of generating a new index-(N+1) file, we instead set the newest index generation to -1 and thus tried to create `index-0` + * If `index-0` is corrupt, this prevents us from ever creating a new snapshot using the broken shard, because we are unable to create `index-0` since it already exists + * Fixed by still using the index generation for naming the next index file, even if it was a broken index file + * Added test that makes sure restoring as well as snapshotting on top of the broken shard index file work as expected + * closes #41304 + +commit 37ebefb553477818ce9081239ad2e755693c064c +Author: James Rodewig +Date: Wed Apr 24 12:18:52 2019 -0400 + + [DOCS] Standardize docs for `url` setting (#41117) + +commit 13d720f4a7ea5d01420903f01062f8f17ce046ea +Author: Hendrik Muhs +Date: Wed Apr 24 16:45:38 2019 +0200 + + [ML-DataFrame] Replace Streamable w/ Writeable (#41477) + + replace usages of Streamable with Writeable + + Relates to #36176 + +commit 07d36fdb23195709362a22a4b2d4597d82870e1d +Author: Benjamin Trent +Date: Wed Apr 24 09:56:59 2019 -0500 + + [ML] refactoring the ML plugin to use the common auditor code (#41419) (#41485) + +commit 381b8e2ece737d5e5ba82d15bf3c647d7336b14d +Author: Armin Braun +Date: Wed Apr 24 13:46:32 2019 +0200 + + Fix BulkProcessor Retry ITs (#41338) (#41472) + + * The test fails for the retry backoff enabled case because the retry handler in the bulk processor hasn't been adjusted to account for #40866 which now might lead to an outright rejection of the request instead of its items individually + * Fixed by adding retry functionality to the top level request as well + * Also fixed the duplicate test for the HLRC that wasn't handling the non-backoff case yet the same way the non-client IT did + * closes #41324 + +commit 2e4ac178e2b414af42cac1c53490143bfceb514d +Author: Armin Braun +Date: Wed Apr 24 12:24:05 2019 +0200 + + Fix Repository Base Path Matching in Azure ITs (#41457) (#41469) + + * Added quotes so that "regexy" base paths like `7.0` that we use on CI + don't break matching + * closes #41405 + +commit 58dd88329f6c222422ea14c7ddb71abdfbcf0ecb +Author: Alpar Torok +Date: Tue Apr 23 14:54:57 2019 +0300 + + Dix the packer cache script + +commit 38e6dcd38800ea7c1caff73e30289bf72a9cc896 +Merge: 85c4cc7f4b4 a74ba7d5ba2 +Author: Michael Basnight +Date: Tue Apr 23 20:38:28 2019 -0500 + + Merge remote-tracking branch 'upstream/7.x' into enrich-7.x + +commit 85c4cc7f4b4d46b2538cf638435b3f9cbc4bb618 +Author: Michael Basnight +Date: Tue Apr 23 13:28:06 2019 -0500 + + Refactor the enrich store to remove it from guice (#41421) + + There is no need to create a enrich store component for the transport + layer since the inner components of the store are either present in the + master node calls or via an already injected ClusterService. This commit + cleans up the class, adds the forthcoming delete call and tests the new + code. + +commit a74ba7d5ba200c6e7967f7066c603bdd5a1ac011 +Author: Jack Conradson +Date: Tue Apr 23 09:10:47 2019 -0700 + + Task to generate Painless API's per context (#41233) + + This adds a gradle task called generateContextDoc in the Painless module. The + task will start a cluster, issue commands against the context rest api for + Painless, and generate documentation for each API per context. Each context + has a first page of classes sorted by package first and class name second, + along with a page per package with each classes' constructors, methods, and + fields. A link is generated for each constructor, method, and field to a JavaDoc + page when possible. + +commit d1a3ae22689a2c45c59dac19b49a3e7c9d5e5c5e +Author: Nik Everett +Date: Tue Apr 23 14:19:26 2019 -0400 + + Docs: Drop last inline callouts + + Drops some inline callouts that snuck into 7.x. We're doings this in + preparation for switching the elasticsearch reference to asciidoctor + which doesn't support them. + +commit 48a17d576823400cd54b535247f9827f8205964e +Author: Guilherme Ferreira +Date: Tue Apr 23 19:12:41 2019 +0200 + + [Docs] Correct default stop list constant (#41342) + +commit 122e727d282c9f9655b2fd7482777ee679b4f90e +Author: James Rodewig +Date: Tue Apr 23 12:41:22 2019 -0400 + + [DOCS] Fix broken link to elasticsearch-php security page + +commit 65af47eb316e141a017b0b0add5167ce553ae725 +Author: Jason Tedor +Date: Tue Apr 23 11:55:27 2019 -0400 + + Introduce aliases version (#41397) + + This commit introduces aliases versions to index metadata. This will be + useful in CCR when we replicate aliases. + +commit 7c2e151bf21f48012a1870c9aa3a39c325f8849c +Author: Nik Everett +Date: Tue Apr 23 10:57:17 2019 -0400 + + Docs: Rework close ml section (#41435) + + Reworks the close ml section to work properly in Asciidoctor. It + renders a little funny in AsciiDoc but AsciiDoc is on its way out + anyway.... + +commit 7e2aec022d8079c01b3917fec8f307fb3e4ba137 +Author: David Roberts +Date: Tue Apr 23 10:37:29 2019 +0100 + + [TEST] Mute BulkRejectionIT.testBulkRejectionAfterDynamicMappingUpdate + + Due to https://github.com/elastic/elasticsearch/issues/41363 + +commit 411994b489e9dba1067b3e8dd1884de5e981c7a9 +Author: David Turner +Date: Tue Apr 23 15:26:14 2019 +0100 + + Mention the cost of tracking live docs in scrolls (#41375) + + Relates #41337, in which a heap dump shows hundreds of MBs allocated on the + heap for tracking the live docs for each scroll. + +commit e2f8ffdde8d2415b71809ed035c98b5450c36528 +Author: Benjamin Trent +Date: Tue Apr 23 09:32:57 2019 -0500 + + [ML][Data Frame] Moving destination creation to _start (#41416) (#41433) + + * [ML][Data Frame] Moving destination creation to _start + + * slight refactor of DataFrameAuditor constructor + +commit 85b9dc18a7f0eea1c919194c5644f3f335bf2d89 +Author: Martijn Laarman +Date: Tue Apr 23 16:23:20 2019 +0200 + + fix #35262 define deprecations of API's as a whole and urls (#39063) + + * fix #35262 define deprecations of API's as a whole and urls + + * document hot threads deprecated paths + + * deprecate scroll_id as part of the URL, documented only as part of the body which is a safer behaviour as well + + * use version numbers up to patch version + + * rest spec parser picks up deprecated paths as paths too + + (cherry picked from commit 7e06023e7603b7584bfd9ee4e8a1ccd82c208ce7) + +commit d8a2970fa40d34676242d520502fcd00a2a2fbfa +Author: David Roberts +Date: Tue Apr 23 15:11:38 2019 +0100 + + [TEST] Mute RemoteClusterServiceTests.testCollectNodes + + Due to https://github.com/elastic/elasticsearch/issues/41067 + +commit e991175776a6487602a8c098771df17dd7ff16bb +Author: Marios Trivyzas +Date: Tue Apr 23 16:30:37 2019 +0300 + + SQL: Implement IIF(, , ) (#41420) + + Implement a more trivial case of the CASE expression which is + expressed as a traditional function with 2 or 3 arguments. e.g.: + + IIF(a = 1, 'one', 'many') + IIF(a > 0, 'positive') + Closes: #40917 + + (cherry picked from commit add02f4f553ad472026dcc1eaa84245a0558a4b0) + +commit 31c609db910e1dfe5dcc8e53f4802c5c5469a8f2 +Author: Hendrik Muhs +Date: Tue Apr 23 13:38:35 2019 +0200 + + [DOCS] change source and dest to be nested and add a query to the example (#41293) + + update source and dest to reflect the changes done in #40396 and add a query to the example. + +commit 4a288af85ff21a94dd24bc3b79f74f03ce65b29a +Author: Jason Tedor +Date: Mon Apr 22 21:46:32 2019 -0400 + + Avoid concurrent modification in mock log appender (#41424) + + It can be the case that while we are setting up expectations that also a + log message is appended. For example, if we are setting up these + expectations after a cluster has formed and messages start being sent + around the cluster. In this case, we would hit a concurrent modification + exception while we are mutating the expectations, and also while the + expectations are being iterated over as a message is appended. This + commit avoids this by using a copy-on-write array list which is safe for + concurrent modification and iteration. Note that another possible + approach here is to use synchronized, but that seems unnecessary since + we don't appear to rely on messages that are sent while we are setting + up expectations. Rather, we are setting up some expectations and some + situation that we think will cause those expectations to be met. Using + copy-on-write array list here is nice since we avoid bottlenecking these + tests on synchronizing these methods. + +commit e6c24e3d2df2f19027ae578ff55ec7088ca27ada +Author: Melori Arellano +Date: Mon Apr 22 14:38:53 2019 -0600 + + [DOCS] Add missing setting skip_unavailable to example + + The example to delete a remote cluster is missing the `skip_unavailable` setting which results in an error: + ``` + "type": "illegal_argument_exception", + "reason": "missing required setting [cluster.remote.tiny-test.seeds] for setting [cluster.remote.tiny-test.skip_unavailable]" + ``` + +commit cc867e8a5a148f6fa0ccd8e11290c64775c2b73c +Author: Ryan Ernst +Date: Mon Apr 22 09:57:51 2019 -0700 + + Remove /var/run/elasticsearch from packages (#41102) + + The pid dir for both systemd and init.d is already managed by those + respective systems (tmpfiles.d and the init script, respectively). Since + the /var/run dir is often mounted as tmpfs, it does not make sense to + have the elasticsearch pid dir added by the package installation. This + commit removes that empty dir from deb and rpm. + +commit 67d4e399c274e4123b4ba7ac90fe9438994d803e +Author: Marios Trivyzas +Date: Mon Apr 22 19:26:15 2019 +0300 + + SQL: Implement CASE... WHEN... THEN... ELSE... END (#41349) + + Implement the ANSI SQL CASE expression which provides the if/else + functionality common to most programming languages. + + The CASE expression can have multiple WHEN branches and becomes a + powerful tool for SQL queries as it can be used in SELECT, WHERE, + GROUP BY, HAVING and ORDER BY clauses. + + Closes: #36200 + (cherry picked from commit 8b2577406f47ae60d15803058921d128390af0b6) + +commit eb2295ac818e37dafb061cfa3d0a3c54b1d601bd +Author: Dimitris Athanasiou +Date: Mon Apr 22 17:42:13 2019 +0300 + + [7.1][ML] Refactor autodetect service into its own class (#41378) (#41409) + + This also improves aims to improve the corresponding unit tests + with regard to readability and maintainability. + +commit 860e783f149faf002ca2c4146ce77cffbae0c785 +Merge: d99249c6246 d2a418152db +Author: Michael Basnight +Date: Mon Apr 22 09:39:28 2019 -0500 + + Merge remote-tracking branch 'upstream/7.x' into enrich-7.x + +commit d2a418152dbf6b66145db7b94e78fd0869980752 +Author: James Rodewig +Date: Mon Apr 22 06:33:55 2019 -0700 + + [DOCS] Remove inline callouts for Asciidoctor migration (#41309) + +commit 3c60f967af6096026b9a3edb22083de8cf777d62 +Author: James Rodewig +Date: Mon Apr 22 06:22:41 2019 -0700 + + [DOCS] Remove inline callouts in SQL Command docs for Asciidoctor migration (#41276) + +commit 490ad2af46ac47a46a3212edb96a86ec8a1cc262 +Author: James Rodewig +Date: Mon Apr 22 06:07:53 2019 -0700 + + [DOCS] Remove inline callouts for Asciidoctor migration (#41266) + +commit b8d054e73be611d1022be6ef2d909b3430a1886c +Author: Ioannis Kakavas +Date: Mon Apr 22 15:38:41 2019 +0300 + + OpenID Connect realm settings and rest API docs (#40740) + + This commit adds the relevant docs for the OpenID Connect + realm settings and the REST APIs that are exposed. + +commit d7cddf815740c8177802caf3eb40d598255aa66a +Author: James Rodewig +Date: Mon Apr 22 05:39:04 2019 -0700 + + [DOCS] Replace nested open block for Asciidoctor migration (#41168) + + * [DOCS] Fix nested open blocks for Asciidoctor migration + + * [DOCS] Reformat table to definitions + +commit c7a74938049564cdf30028e7d52fa69c82f60a17 +Author: James Rodewig +Date: Mon Apr 22 05:26:36 2019 -0700 + + [DOCS] Remove abbrevtitles for Asciidoctor migration (#41366) + +commit c3e0ae24d377f9fccd6edb326fee995dd2e6a8b4 +Author: Albert Zaharovits +Date: Mon Apr 22 10:25:24 2019 +0300 + + Fix role mapping DN field wildcards for users with NULL DNs (#41343) + + The `DistinguishedNamePredicate`, used for matching users to role mapping + expressions, should handle users with null DNs. But it fails to do so (and this is + a NPE bug), if the role mapping expression contains a lucene regexp or a wildcard. + + The fix simplifies `DistinguishedNamePredicate` to not handle null DNs at all, and + instead use the `ExpressionModel#NULL_PREDICATE` for the DN field, just like + any other missing user field. + +commit 5c40fc9ba5e470b13d04882e9653aa772fd45039 +Author: Jason Tedor +Date: Sat Apr 20 08:22:56 2019 -0400 + + Remove script engine from X-Pack plugin (#41387) + + The X-Pack plugin implements ScriptEngine yet it does not actually + implement any of the methods on the interface, effectively making this a + no-op. This commit removes this interface from the X-Pack plugin. + +commit 21bf2fe3c4a6c6cc92d8c18457e28f68dca051b3 +Author: Jason Tedor +Date: Sat Apr 20 08:21:10 2019 -0400 + + Reduce security permissions in CCR plugin (#41391) + + It looks like these permissions were copy/pasted from another plugin yet + almost none of these permissions are needed for the CCR plugin. This + commit removes all these unneeded permissions from the CCR plugin. + +commit 2b20bd0b8d9a6c05b85e6b0ab318a501acca7549 +Author: Joe Zack +Date: Sat Apr 20 04:32:35 2019 -0400 + + Fix discovery config in docker-compose docs (#41394) + + Today's `docker-compose` docs are missing the `discovery.seed_nodes` config on + one of the nodes. With today's configuration the cluster can still form the + first time it is started, because `cluster.initial_master_nodes` requires both + nodes to bootstrap the cluster which ensures that each discover the other. + However if `es02` is elected master it will remove `es01` from the voting + configuration and then when restarted it will form a cluster on its own without + needing to do any discovery. Meanwhile `es01` doesn't know how to find `es02` + after a restart so will be unable to join this cluster. + + This commit fixes this by adding the missing configuration. + +commit ac7d5e3e9bab9c79a0414e85e05bfec5a9afe19b +Author: Jason Tedor +Date: Fri Apr 19 17:23:15 2019 -0400 + + Fix reference to ignore_above from analyis-icu docs + + This commit fixes a reference to the docs for ignore_above from the + analysis-icu plugin docs. + +commit c80f86e3e4d94dc4220a7b268514282bc5acbef2 +Author: clement-tourriere +Date: Fri Apr 19 22:17:00 2019 +0200 + + Add ignore_above in ICUCollationKeywordFieldMapper (#40414) + + Add the possibility to use ignore_above parameter in ICUCollationKeywordFieldMapper. + + Close #40413 + +commit 38b43c4a56fa79f0972a50f0862c1761830ec095 +Author: Jason Tedor +Date: Fri Apr 19 17:14:49 2019 -0400 + + Limit the number of forks getting Java versions (#41386) + + To reduce configuration time, we fork some threads to compute the Java + version for the various configured Javas. However, as the number of + JAVA${N}_HOME variable increases, the current implementation creates as + many threads as there are such variables, which could be more than the + number of physical cores on the machine. It is not likely that we would + see benefits to trying to execute all of these once beyond the number of + physical cores (maybe simultaneous multi-threading helps though, who + knows. Therefore, this commit limits the parallelization here to the + number number of physical cores. + +commit c0164cbb637570659a4316f0c69b45dfc93f8ac9 +Author: Ryan Ernst +Date: Fri Apr 19 11:34:16 2019 -0700 + + Only include relevant platform files from modules (#41089) + + This commit adds a filter to the files include from modules to only + include platform specific files relevant to the distribution being + built. For example, the deb files on linux would now only include linux + ML binaries, and not windows or macos files. + +commit 154d40494f36415f36643784d2de04c7f097e590 +Author: Jason Tedor +Date: Fri Apr 19 14:08:22 2019 -0400 + + Fix build issue if no specific Java version are set (#41379) + + If no Java versions are set then when we size the executor thread pool + we end up with zero threads, which is illegal. This commit avoids that + problem by only starting the executor when needed. + +commit bcd0939b611a26b3cddee5c849fba203e72ddc0d +Author: Ryan Ernst +Date: Fri Apr 19 09:49:36 2019 -0700 + + Add a rule for task dependencies (#41322) + + This commit adds a task rule to print the task dependencies of any task. + It only prints the direct dependencies, but makes debugging missing + dependencies a lot easier. + +commit 7a34ba35f7e362a541f52d8dc884118c193db7c9 +Author: Marios Trivyzas +Date: Fri Apr 19 19:03:28 2019 +0300 + + SQL: Fix bug with optimization of null related conditionals (#41355) + + The SimplifyConditional rule is removing NULL literals from those + functions to simplify their evaluation. This happens in the Optimizer + and a new instance of the conditional function is generated. Previously, + the dataType was not set properly (defaulted to DataType.NULL) for + those new instances and since the resolveType() wasn't called again + it resulted in returning always null. + + E.g.: + + SELECT COALESCE(null, 'foo', null, 'bar') + + COALESCE(null, 'foo', null, 'bar') + ----------------- + null + + This issue was not visible before because the tests always used an alias + for the conditional function which caused the resolveType() to be + called which sets the dataType properly. + + E.g.: + + SELECT COALESCE(null, 'foo', null, 'bar') as c + + c + ----------------- + foo + + (cherry picked from commit c39980a65dd593363f1d8d1b038b26cb0ce02aaf) + +commit d989079df5d90c7132531d92d0b793f2b16be9a8 +Author: Arlind +Date: Fri Apr 19 10:24:29 2019 +0200 + + Update glossary.asciidoc (#41364) + +commit e1e2568fa302f8253a0b1c08183640559b1027e7 +Author: Alpar Torok +Date: Fri Apr 19 09:46:21 2019 +0300 + + Add FIPS specific testclusters configuration (#41199) + + ClusterFormationTasks auto configured these properties for clusters. + This PR adds FIPS specific configuration across all test clusters from + the main build script to prevent coupling betwwen testclusters and the + build plugin. + + Closes #40904 + +commit 0bb15d3dac6f2da73df6aca68bf2c389ace26680 +Author: David Turner +Date: Fri Apr 19 07:37:22 2019 +0100 + + Allow ops to be blocked after primary promotion (#41360) + + Today we assert that there are no operations in flight in this test. However we + will sometimes be in a situation where the operations are blocked, and we + distinguish these cases since #41271 causing the assertion to fail. This commit + addresses this by allowing operations to be blocked sometimes after a primary + promotion. + + Fixes #41333. + +commit 4ef4ed66b917634eaf3830455cc0ae5e75860db6 +Author: Alpar Torok +Date: Fri Apr 19 09:34:23 2019 +0300 + + Convert repository-hdfs to testclusters (#41252) + + * Convert repository-hdfs to testclusters + + Relates #40862 + +commit 2f8bbce85da34d34718f674c5e425d186d271af4 +Author: Alpar Torok +Date: Fri Apr 19 09:19:28 2019 +0300 + + Clean up build tool dependencies (#41336) + + We are no longer using these dependencies. + + Relates to #41061 since the class that seems to be leaking is both part + of Gradle and the logging jar. + +commit 1f91759bc7d0749f4872bbe362906b4714c4b936 +Author: Alpar Torok +Date: Fri Apr 19 08:55:10 2019 +0300 + + Improove the configuration time if the build (#41251) + + This will help with reproduction lines and running tests form IDEs and + other operations that are quick and executed often enough for the + configuration time to matter. + + Running Gradle with a FIPS JVM is not supproted, so if the runtime JVM + is the same one, no need to spend time checking for fips support. + + Verification of the JAVA_HOME env vars is now async and + parallel so it doesn't block configuration. + +commit c03394c236f36f6a3d441dfad3a6cbd9280dedb2 +Author: Lisa Cawley +Date: Thu Apr 18 17:30:08 2019 -0700 + + [DOCS] Fixes deprecation notice in pagerduty action (#41362) + +commit 8f73e1e883e0da94359d8d581603d1608fbb38b9 +Author: Jim Ferenczi +Date: Thu Apr 18 22:48:22 2019 +0200 + + Fix unmapped field handling in the composite aggregation (#41280) + + The `composite` aggregation maps unknown fields as numerics, this means that + any `after` value that is set on a query with an unmapped field on some indices + will fail if the provided value is not numeric. This commit changes the default + value source to use keyword instead in order to be able to parse any type of after + values. + +commit 754037b71e9d783bf4d2d5cdea5538b2a17bb348 +Author: Jim Ferenczi +Date: Thu Apr 18 22:30:51 2019 +0200 + + Unified highlighter should ignore terms that targets the _id field (#41275) + + The `_id` field uses a binary encoding to index terms that is not compatible with + the utf8 automaton that the unified highlighter creates to reanalyze the input. + For these reason this commit ignores terms that target the `_id` field when + `require_field_match` is set to false. + + Closes #37525 + +commit 068f8ba223376d7c1a2e30dec953b6a750ed9006 +Author: Jim Ferenczi +Date: Thu Apr 18 22:29:36 2019 +0200 + + more_like_this query to throw an error if the like fields is not provided (#40632) + + With the removal of the `_all` field the `mlt` query cannot infer a field name + to use to analyze the provided (un)like text if the `fields` parameter is not + explicitly set in the query and the `index.query.default_field` is not changed + in the index settings (by default it is set to `*`). For this reason the like text + is ignored and queries are only built from the provided document ids. + This change fixes this bug by throwing an error if the fields option is not set + and the `index.query.default_field` is equals to `*`. The error is thrown only + if like or unlike texts are provided in the query. + +commit 1f5cd410b821e426c144f71ea970b333ee150163 +Author: Jay Modi +Date: Thu Apr 18 14:17:16 2019 -0600 + + Clean up docs regarding recommended JVM (#41356) + + This change clarifies the documentation around the recommended JVM. The + recommended JVM is the bundled JVM. If a user does not use our + recommended JVM we suggest that they use a supported LTS version of the + JVM. + + Closes #41132 + +commit 5f497d427496b90b1dbd9263aeb1093158c62cfe +Author: Costin Leau +Date: Thu Apr 18 22:44:16 2019 +0300 + + SQL: Predicate diff takes into account all values (#41346) + + Fix bug in predicate subtraction that caused the evaluation to be + skipped on the first mismatch instead of evaluating the whole list. In + some cases this caused not only an incorrect result but one that kept on + growing causing the engine to bail + + Fix #40835 + + (cherry picked from commit bd2b33d6eaca616a5acd846204e2d12f905854d4) + +commit cfed5d65bee8d16ee7822cf6ec593dac18b26c43 +Author: Andrei Stefan +Date: Thu Apr 18 20:09:59 2019 +0300 + + SQL: fix *SecurityIT tests by covering edge case scenarios when audit file rolls over at midnight (#41328) + + * Handle the scenario where assertLogs() is not called from a test method + but the audit rolling file rolls over. + * Use a local boolean variable instead of the static one to account for + assertBusy() code block possibly being called multiple times and having + different execution paths. + + (cherry picked from commit 6f642196cbab90079c610097befc794746170df1) + +commit 0227ac5690fb78341b521056d5fac1bb3f129812 +Author: Mark Vieira +Date: Thu Apr 18 11:15:34 2019 -0700 + + Fix issue with subproject test task dependencies (#41321) (#41351) + +commit 11dc9fe249dd609ec986d5d887a67926c3b5d244 +Author: Simon Willnauer +Date: Thu Apr 18 19:14:23 2019 +0200 + + Mark searcher as accessed in acquireSearcher (#41335) + + This fixes an issue where every N seconds a slow search request is triggered + since the searcher access time is not set unless the shard is idle. This change + moves to a more pro-active approach setting the searcher as accessed all the time. + +commit 6b4cf8f0bdcacf2c39286c4ab3e39628f61f983e +Author: Nik Everett +Date: Thu Apr 18 09:21:13 2019 -0400 + + Docs: Revert accidental gcs docs change + + I snuck an extra change in my last commit accidentally. This reverts it. + +commit 0a343be90c29521539bb146b044d606177989e44 +Author: Nik Everett +Date: Thu Apr 18 09:17:22 2019 -0400 + + Docs: Fix deprecation warning in Asciidoctor + + Fix a deprecation warning that wasn't rendering correctly in + asciidoctor. This one needed to be explicitly marked as an inline macro + because it is on its own line and it needed to have its text escaped + because it contained a `,`. It also was missing explanitory text for + what the setting was. + +commit a699cb76a5cde8b9e1252605d3f3c98a4304e38c +Author: Adrien Grand +Date: Thu Apr 18 14:41:09 2019 +0200 + + Fix javadoc tag. (#41330) + + s/returns/return/ + +commit 389a13b68e3b696ecde4ce25df76d4eaa00db898 +Author: Armin Braun +Date: Thu Apr 18 11:55:28 2019 +0200 + + Mute BulkProcessorRetryIT#testBulkRejectionLoadWithBackoff (#41325) (#41331) + + * For #41324 + +commit 86e56590a7d6b0f6551b18d6c950f63dc135d677 +Author: Adrien Grand +Date: Thu Apr 18 11:31:00 2019 +0200 + + Revert "Disable CcrRetentionLeaseIT#testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes." + + This reverts commit 343039e20053aeb42a26438078ae2dc6472ef9c3. + +commit 343039e20053aeb42a26438078ae2dc6472ef9c3 +Author: Adrien Grand +Date: Thu Apr 18 11:18:25 2019 +0200 + + Disable CcrRetentionLeaseIT#testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes. + + Relates #39331. + +commit d99249c62461c690a70405ec5eefc4e663851b0d +Author: Martijn van Groningen +Date: Wed Apr 17 17:54:10 2019 +0200 + + Move the policy class to xpack core module. (#41311) + + This allows the transport client use this class in enrich APIs. + + Relates to #40997 + +commit 45c151ff8bd1428db8f52455f5cad43a03a75346 +Author: Alpar Torok +Date: Thu Apr 18 09:50:49 2019 +0300 + + Disable composePull only if it exists (#41306) + + The task will not be created when docker is not available. + +commit a4a4259cac97ffbfc54907a5601d74105b3e90a8 +Author: Alpar Torok +Date: Thu Apr 18 09:25:30 2019 +0300 + + Mute failing test + + Tracking #41326 + +commit 44a0c468cf7da451ee572ccdfb3913283efbe16f +Author: Alpar Torok +Date: Thu Apr 18 08:51:10 2019 +0300 + + Clean up clusters between tests (#41187) + + This PR adds additional cleanup when stopping the node. + The data dir is excepted because it gets reused in some tests. + Without this cleanup the number of working dir copies could grew to + exhaust all available disk space. + +commit c77e10b16be8f91a6547a02777e9b9dfae8bd549 +Author: Armin Braun +Date: Thu Apr 18 07:10:23 2019 +0200 + + Handle Bulk Requests on Write Threadpool (#40866) (#41315) + + * Bulk requests can be thousands of items large and take more than O(10ms) time to handle => we should not handle them on the transport threadpool to not block select loops + * relates #39128 + * relates #39658 + +commit 2523ad03cafc6aaa96d2a0762672730aa7d59aed +Author: Bob Blank <39975390+BobBlank12@users.noreply.github.com> +Date: Wed Apr 17 16:06:03 2019 -0500 + + Fixing missing link on session_token (#41314) + + session_token was missing a "secure link" + +commit 946baf87d31e22cffda6a20d6f5a0bd1f779918c +Author: David Turner +Date: Wed Apr 17 19:54:55 2019 +0100 + + Assert TransportReplicationActions acquire permits (#41271) + + Today we do not distinguish "no operations in flight" from "operations are + blocked", since both return `0` from `IndexShard#getActiveOperationsCount()`. + We therefore cannot assert that every `TransportReplicationAction` performs its + actions under permit(s). This commit fixes this by returning + `IndexShard#OPERATIONS_BLOCKED` if operations are blocked, allowing these two + cases to be distinguished. + +commit 66366d0307e7255be31f0e30dabbcfd4c96a7a8e +Author: Gordon Brown +Date: Wed Apr 17 13:42:36 2019 -0600 + + Extract template management from Watcher (#41169) + + This commit extracts the template management from Watcher into an + abstract class, so that templates and lifecycle policies can be managed + in the same way across multiple plugins. This will be useful for SLM, as + well as potentially ILM and any other plugins which need to manage index + templates. + +commit 7e62ff28237f170a0476ee69478943c563b518c9 +Author: Zachary Tong +Date: Wed Apr 17 13:33:51 2019 -0400 + + [Rollup] Validate timezones based on rules not string comparision (#36237) + + The date_histogram internally converts obsolete timezones (such as + "Canada/Mountain") into their modern equivalent ("America/Edmonton"). + But rollup just stored the TZ as provided by the user. + + When checking the TZ for query validation we used a string comparison, + which would fail due to the date_histo's upgrading behavior. + + Instead, we should convert both to a TimeZone object and check if their + rules are compatible. + +commit 4d964194db8399fc34ab9e4a99f4538d542753bb +Author: Christoph Büscher +Date: Wed Apr 17 18:44:07 2019 +0200 + + Fix error applying `ignore_malformed` to boolean values (#41261) + + The `ignore_malformed` option currently works on numeric fields only when the + bad value isn't a string value but not if it is a boolean. In this case we get a + parsing error from the xContent parser which we need to catch in addition to the + field mapper. + + Closes #11498 + +commit 2670ed2f8f4ce136952fb01f00535fc0f1fc5267 +Author: David Turner +Date: Wed Apr 17 16:36:04 2019 +0100 + + Assert the stability of custom search preferences (#41150) + + Today the `?preference=custom_string_value` search preference will only change + its choice of a shard copy if something changes the `IndexShardRoutingTable` + for that specific shard. Users can use this behaviour to route searches to a + consistent set of shard copies, which means they can reliably hit copies with + hot caches, and use the other copies only for redundancy in case of failure. + However we do not assert this property anywhere, so we might break it in + future. + + This commit adds a test that shows that searches are routed consistently even + if other indices are created/rebalanced/deleted. + + Relates https://discuss.elastic.co/t/176598, #41115, #26791 + +commit e090176f17867c8c9a04986ebc1b3516d63d0dc2 +Author: Iana Bondarska +Date: Wed Apr 17 17:01:46 2019 +0200 + + [ML] Exclude analysis fields with core field names from anomaly results (#41093) + + Added "_index", "_type", "_id" to list of reserved fields. + + Closes #39406 + +commit 2ee87c99d9eb2a37e601e58a7ac858744dd94a45 +Author: Nhat Nguyen +Date: Wed Apr 17 10:25:47 2019 -0400 + + Fix bwc version of sanity check of read only engine + + Relates #41041 + +commit aa0c957a4a3ba611db9b9fd494e56345a4ed0936 +Author: Nhat Nguyen +Date: Thu Apr 11 15:15:00 2019 -0400 + + Do not trim unsafe commits when open readonly engine (#41041) + + Today we always trim unsafe commits (whose max_seq_no >= global + checkpoint) before starting a read-write or read-only engine. This is + mandatory for read-write engines because they must start with the safe + commit. This is also fine for read-only engines since most of the cases + we should have exactly one commit after closing an index (trimming is a + noop). However, this is dangerous for following indices which might have + more than one commits when they are being closed. + + With this change, we move the trimming logic to the ctor of InternalEngine + so we won't trim anything if we are going to open a read-only engine. + +commit f7e590ce0d7a2b2c17eb642cee78e31d74ef60b0 +Author: Adrien Grand +Date: Wed Apr 17 16:11:14 2019 +0200 + + ProfileScorer should propagate `setMinCompetitiveScore`. (#40958) (#41302) + + Currently enabling profiling disables top-hits optimizations, which is + unfortunate: it would be nice to be able to notice the difference in method + counts and timings depending on whether total hit counts are requested. + +commit 9fd5237fd453db6d696e642d1f9a2f595f81bf03 +Author: Adrien Grand +Date: Wed Apr 17 16:10:53 2019 +0200 + + Clean up Node#close. (#39317) (#41301) + + `Node#close` is pretty hard to rely on today: + - it might swallow exceptions + - it waits for 10 seconds for threads to terminate but doesn't signal anything + if threads are still not terminated after 10 seconds + + This commit makes `IOException`s propagated and splits `Node#close` into + `Node#close` and `Node#awaitClose` so that the decision what to do if a node + takes too long to close can be done on top of `Node#close`. + + It also adds synchronization to lifecycle transitions to make them atomic. I + don't think it is a source of problems today, but it makes things easier to + reason about. + +commit 1d2365f5b6dc774487f3622052ec32a3fa2f5962 +Author: David Kyle +Date: Wed Apr 17 14:56:30 2019 +0100 + + [ML-DataFrame] Refactorings and tidying (#41248) + + Remove unnecessary generic params from SingleGroupSource + and unused code from the HLRC + +commit 6566979c189dc111ea0128469771aafb3c3c556e +Author: Jason Tedor +Date: Wed Apr 17 06:59:29 2019 -0400 + + Always check for archiving broken index settings (#41209) + + Today we check if an index has broken settings when checking if an index + needs to be upgraded. However, it can be the case that an index setting + became broken even if an index is already upgraded to the current + version if the user removed a plugin (or downgraded from the default + distribution to the non-default distribution) while on the same version + of Elasticsearch. In this case, some registered settings would go + missing and the index would now be broken. Yet, we miss this check and + instead of archiving the settings, the index becomes unassigned due to + the missing settings. This commit addresses this by checking for broken + settings whether or not the index is upgraded. + +commit badb7a22e0b663693bab0e79d207d94781389ce7 +Author: Christoph Büscher +Date: Wed Apr 17 10:21:44 2019 +0200 + + Some cleanups in NoisyChannelSpellChecker (#40949) + + One of the two #getCorrections methods is only used in tests, so we can move + it and any of the required helper methods to that test. Also reducing the + visibility of several methods to package private since the class isn't used + elsewhere outside the package. + +commit 711d2545aa80d434d384745067f6e300e12add7c +Author: David Kyle +Date: Wed Apr 17 09:01:15 2019 +0100 + + [ML-DataFrame] Resolve random test failure using deterministic name (#41262) + +commit bfa06d963e6161c337d2b40b41295f42379aeeeb +Author: David Turner +Date: Wed Apr 17 08:34:07 2019 +0100 + + Do not create missing directories in readonly repo (#41249) + + Today we erroneously look for a node setting called `readonly` when deciding + whether or not to create a missing directory in a filesystem repository. This + change fixes this by using the repository setting instead. + + Closes #41009 + Relates #26909 + +commit 3fd081528dbd29d573644949fee974869f624cca +Author: Marios Trivyzas +Date: Wed Apr 17 10:12:11 2019 +0300 + + SQL: Allow current_date/time/timestamp to be also used as a function escape pattern (#41254) + + CURRENT_DATE/CURRENT_TIME/CURRENT_TIMESTAMP can be used as SQL keywords + (without parentheses) and therefore there is a special rule in the + grammar to accommodate this. + + Previously, this rule was also catching the parenthesised version of those functions too, + not allowing the {fn ()} to be used. E.g.: + {fn current_time(2)} or {fn current_timestamp()} + + Now, the grammar rule catches only the keyword versions and all the parenthesised + versions go through the normal function resolution. As a consequence the validation + of the precision is moved from the parser lever (ExpressionBuilder) to the function + implementations. + + Fixes: #41240 + (cherry picked from commit bfbc9f140144b5a35aa29008b58bf58074419853) + +commit 84e2f9d8ecb834b08e8188c81e8f0ac76fa93e61 +Author: Alpar Torok +Date: Wed Apr 17 08:58:11 2019 +0300 + + fix the packer cache script (#41183) + + * fix the packer cache script + + This PR disabled the explicit pull since it seems this always tries to + work with a registry. + Functionality will not be affected since we will still build the images + on pull. + +commit 0d1178fca69c13cb17f09fc42393103b3cd8e742 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Apr 17 14:25:33 2019 +1000 + + put mapping authorization for alias with write-index and multiple read indices (#40834) (#41287) + + When the same alias points to multiple indices we can write to only one index + with `is_write_index` value `true`. The special handling in case of the put + mapping request(to resolve authorized indices) has a check on indices size + for a concrete index. If multiple indices existed then it marked the request + as unauthorized. + + The check has been modified to consider write index flag and only when the + requested index matches with the one with write index alias, the alias is considered + for authorization. + + Closes #40831 + +commit 6a552c05fef4bc5aa963d362e2d44483e62b3b6b +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Apr 17 14:15:05 2019 +1000 + + Use alias name from rollover request to query indices stats (#40774) (#41284) + + In `TransportRolloverAction` before doing rollover we resolve + source index name (write index) from the alias in the rollover request. + Before evaluating the conditions and executing rollover action, we + retrieve stats, but to do so we used the source index name + resolved from the alias instead of alias from the index. + This fails when the user is assigned a role with index privilege on the + alias instead of the concrete index. This commit fixes this by using + the alias from the request. + After this change, verified that when we retrieve all the stats (including write + read indexes) + we are considering only source index. + + Closes #40771 + +commit 85912b89febbcd57052da02a001c29d93c218657 +Author: Costin Leau +Date: Tue Apr 16 22:40:29 2019 +0300 + + SQL: Fix LIMIT bug in agg sorting (#41258) + + When specifying a limit over an agg sorting, the limit will be pushed + down to the grouping which affects the custom sorting. This commit fixes + that and restricts the limit only to sorting. + + Fix #40984 + + (cherry picked from commit da3726528d9011b05c0677ece6d11558994eccd9) + +commit 7d5ff5a1fabb2d98a9e3c8c7e4953fe1dff16f6f +Author: Nik Everett +Date: Tue Apr 16 15:27:51 2019 -0400 + + Docs: Drop inline callouts from two SQL pages (#41270) + + Drops inline callouts from the docs for SQL's string and type-conversion + functions because they are not compatible with Asciidoctor. + +commit c4ffd758058084502e7a9b6e261dc6a45f64d357 +Author: Marios Trivyzas +Date: Tue Apr 16 20:39:54 2019 +0300 + + SQL: Translate MIN/MAX on keyword fields as FIRST/LAST (#41247) + + Although the translation rule was implemented in the `Optimizer`, + the rule was not added in the list of rules to be executed. + + Relates to #41195 + Follows #37936 + + + (cherry picked from commit f426a339b77af6008d41cc000c9199fe384e9269) + +commit a01dd96afec31ac9b8dd5a19e0c110ce08b7e772 +Author: Nik Everett +Date: Tue Apr 16 13:47:39 2019 -0400 + + Drop inline callouts from SQL conditional docs (#41205) + + Drops "inline callouts" from the docs for SQL conditionals because they + aren't supported by Asciidoctor. + + Relates to #41128 + +commit 02247cc7df46da7e233410f613c30607d1dd7be3 +Author: Hendrik Muhs +Date: Tue Apr 16 18:53:36 2019 +0200 + + [ML-DataFrame] adapt page size on circuit breaker responses (#41149) + + handle circuit breaker response and adapt page size to reduce memory pressure, reduce preview buckets to 100, initial page size to 500 + +commit 043c1f5d428f11f9303e43058d5a61685a638608 +Author: Jim Ferenczi +Date: Tue Apr 16 18:16:55 2019 +0200 + + Unified highlighter should respect no_match_size with number_of_fragments set to 0 (#41069) + + The unified highlighter returns the first sentence of the text when number_of_fragments + is set to 0 (full highlighting). This is a legacy of the removed postings highlighter + that was based on sentence break only. This commit changes this behavior in order + to respect the provided no_match_size value when number_of_fragments is set to 0. + This means that the behavior will be consistent for any value of the number_of_fragments option. + + Closes #41066 + +commit 9bf8bd40aed95fa0b6f913a6c45eecc454dc8cf2 +Author: Costin Leau +Date: Tue Apr 16 19:49:00 2019 +0300 + + SQL: Tweak pattern matching in SYS TABLES (#41243) + + Yet another improvement to SYS TABLES on differentiating between table + types specified as '%' and '' while maintaining legacy support for null + + Fix #40775 + + (cherry picked from commit 6dbca5edd335eb1da8e7825389a15e5fe45397d4) + +commit 4bff26ef69a1baf8d4be0198d1d47f219e6f9843 +Author: Lisa Cawley +Date: Tue Apr 16 09:08:07 2019 -0700 + + [DOCS] Deprecates transport settings (#41174) + +commit 6cc35d372450dc1d644c99ffd881b807f85e2e1e +Author: David Roberts +Date: Tue Apr 16 09:16:23 2019 +0100 + + [ML] Unmute MachineLearningIT.testDeleteExpiredData (#41186) + + The cause of failure was fixed by elastic/ml-cpp#459, + so all that remains on the Java side is to unmute the + test that was failing. + + Closes #41070 + +commit ec8709e831384298debc0f8a0c4e7afcf3d1c0a0 +Author: Gordon Brown +Date: Tue Apr 16 09:25:51 2019 -0600 + + Check allocation rules are cleared after ILM Shrink (#41170) + + Adds some checks to make sure that the allocation rules that ILM adds + before a shrink are cleared after the shrink is complete + +commit c4e84e2b34b924fcda972615ec707fdd9393f2d3 +Author: Armin Braun +Date: Tue Apr 16 17:19:05 2019 +0200 + + Add Bulk Delete Api to BlobStore (#40322) (#41253) + + * Adds Bulk delete API to blob container + * Implement bulk delete API for S3 + * Adjust S3Fixture to accept both path styles for bulk deletes since the S3 SDK uses both during our ITs + * Closes #40250 + +commit c22a2cea12852a95d8fe570d90487344e1149083 +Author: Jim Ferenczi +Date: Tue Apr 16 13:02:10 2019 +0200 + + BlendedTermQuery should ignore fields that don't exists in the index (#41125) + + Today the blended term query detects if a term exists in a field by looking at the term statistics in the index. + However the value to indicate that a term has no occurence in a field have changed in Lucene. A non-existing term now returns + a doc and total term frequency of 0. Because of this disrepancy the blended term query picks 0 as the minimum frequency for a term + even if other fields have documents for this terms. This confuses the term queries that the blending creates since some of them + contain a custom state that indicates a frequency of 0 even though the term has some occurence in the field. For these terms an exception + is thrown because the term query always checks that the term state's frequency is greater than 0 if there are documents associate to it. + This change fixes this bug by ignoring terms with a doc freq of 0 when the blended term query picks the minimum term frequency among the + requested fields. + + Closes #41118 + +commit 116167df5536b87e42273ba4a722c58c44504892 +Author: David Kyle +Date: Tue Apr 16 13:43:00 2019 +0100 + + [ML] Write header to autodetect before it is visible to other calls (#41085) + +commit 8577bbd73bac68b771c4dcfe39c90e5f519795c1 +Author: David Turner +Date: Tue Apr 16 13:36:29 2019 +0100 + + Inline TransportReplAct#createReplicatedOperation (#41197) + + `TransportReplicationAction.AsyncPrimaryAction#createReplicatedOperation` + exists so it can be overridden in tests. This commit re-works these tests to + use a real `ReplicationOperation` and inlines the now-unnecessary method. + + Relates #40706. + +commit 10e58210a021f4b849c812a06892f281aaa834cd +Author: David Turner +Date: Tue Apr 16 12:49:47 2019 +0100 + + Validate cluster UUID when joining Zen1 cluster (#41063) + + Today we fail to join a Zen2 cluster if the cluster UUID does not match our + own, but we do not perform the same validation when joining a Zen1 cluster. + This means that a Zen2 node will pass join validation and be added to a Zen1 + cluster but will reject all cluster states from the master. + + Relates #37775 + +commit 8ee84f22687d7d9aeff5631f1394a9b25c823f39 +Author: Nhat Nguyen +Date: Tue Apr 16 04:13:47 2019 -0400 + + Correct flush parameters in engine test + + Since #40213, we forbid a combination of flush parameters: force=true + and wait_if_ongoing=false. + + Closes #41236 + +commit f8161ffa88b5784e3ca55898f31c280dcaab4186 +Author: Christoph Büscher +Date: Tue Apr 16 10:03:35 2019 +0200 + + Fix some `range` query edge cases (#41160) + + Currently we throw an error when a range querys minimum value exceeds the + maximum value due to the fact that they are neighbouring values and both upper + and lower value are excluded from the interval. + + Since this is a condition that the user usually doesn't specify conciously (at + least in the case of float and double values its difficult to see which values + are adjacent) we should ignore those "wrong" intervals and create a + MatchNoDocsQuery in those cases. + + We should still throw errors with an actionable message if the user specifies + the query interval in a way that min value > max value. This PR adds those + checks and tests for those cases. + + Closes #40937 + +commit 36a8c7aa0bf04aeeef82f4073ccb3456dd16163e +Author: David Turner +Date: Mon Apr 15 19:22:10 2019 +0100 + + Add 'DO NOT TOUCH' warnings to disco settings docs (#41211) + +commit ad3b7abaa3db753ae0482b26bec2d0fb912b13c6 +Author: Tim Brooks +Date: Mon Apr 15 21:43:09 2019 -0600 + + Deprecate old transport settings (#41229) + + This is related to #36652. We intend to remove a number of old transport + settings in 8.0. This commit deprecates those settings for 7.x. + +commit 750db02b54daf42c1616c605fefe177d592715d7 +Author: Shaunak Kashyap +Date: Mon Apr 15 20:17:05 2019 -0700 + + Expand beats_system role privileges (#40876) (#41232) + + Traditionally we have [recommended](https://www.elastic.co/guide/en/beats/filebeat/current/monitoring.html) that Beats send their monitoring data to the **production** Elasticsearch cluster. Beats do this by calling the `POST _monitoring/bulk` API. When Security is enabled this API call requires the `cluster:admin/xpack/monitoring/bulk` privilege. The built-in `beats_system` role has this privilege. + + [Going forward](https://github.com/elastic/beats/pull/9260), Beats will be able to send their monitoring data directly to the **monitoring** Elasticsearch cluster. Beats will do this by calling the regular `POST _bulk` API. When Security is enabled this API call requires the `indices:data/write/bulk` privilege. Further, the call has to be able to create any indices that don't exist. + + This PR expands the built-in `beats_system` role's privileges. Specifically, it adds index-level `write` and `create_index` privileges for `.monitoring-beats-*` indices. + + This will allow Beats users to continue using the `beats_system` role for the new direct monitoring route when Security is enabled. + +commit 56c00eecbc0697cf102c8f6d6e04b8e66017eae9 +Author: Tim Brooks +Date: Mon Apr 15 16:54:24 2019 -0600 + + Remove string usages of old transport settings (#41207) + + This is related to #36652. We intend to deprecate a number of transport + settings in 7.x and remove them in 8.0. This commit removes the string + usages of these settings. + +commit 7e59794ced8ca858c2fbf0991fe8efb1228eb24b +Author: Gordon Brown +Date: Mon Apr 15 16:20:37 2019 -0600 + + Log every use of ILM Move to Step API (#41171) + + Usage of the ILM Move to Step API can result in some very odd + situations, and for diagnosing problems arising from these situations it + would be nice to have a record of when this API was called with what + parameters. + + Also, adds a dedicated logger for TransportMoveToStepAction, + rather than using the (deprecated) inherited one. + +commit db13043d3bad170a73668323e4bf7729f805bbef +Author: Julie Tibshirani +Date: Tue Apr 9 10:49:01 2019 -0700 + + Some clarifications in the 'enabled' documentation. (#40989) + + This PR makes a few clarifications to the docs for the `enabled` setting: + - Replace references to 'mapping type' with 'mapping' or 'mapping definition'. + - In code examples, clarify that the disabled fields have type `object`. + - Add a section on how disabled fields can hold non-object data. + +commit f56b2ecb3761229408f32c4ce1966c3c3651b3e5 +Author: Martijn van Groningen +Date: Mon Apr 15 19:14:43 2019 +0200 + + Remove xpack dependencies from qa rest modules (#41134) (7.x backport) (#41202) + + This commit removes xpack dependencies of many xpack qa modules. + (for some qa modules this will require some more work) + + The reason behind this change is that qa rest modules should not depend + on the x-pack plugins, because the plugins are an implementation detail and + the tests should only know about the rest interface and qa cluster that is + being tested. + + Also some qa modules rely on xpack plugins and hlrc (which is a valid + dependency for rest qa tests) creates a cyclic dependency and this is + something that we should avoid. Also Eclipse can't handle gradle cyclic + dependencies (see #41064). + + * don't copy xpack-core's plugin property into the test resource of qa + modules. Otherwise installing security manager fails, because it tries + to find the XPackPlugin class. + +commit 636b9338a64eb67c610ded7ad89ea96ca6248e83 +Author: Alpar Torok +Date: Mon Apr 15 18:48:05 2019 +0300 + + Fix the rpm and deb names for version starting with 7.0.0 (#41198) + + With the 7.0.0 release, we switched to download the packages instead of + using locally built ones. + This PR fixes the artifact names to include the architecture as + introduced in the 7.0.0 release. + +commit 454148eee61e98dc905340ab6e6f2124dda18380 +Author: Jason Tedor +Date: Mon Apr 15 11:27:38 2019 -0400 + + Fix intervals section of auto date-histogram docs (#41203) + + This section should be at the same sub-level as other sections in the + auto date-histogram docs, otherwise it is rendered on to another page + and is confusing for users to understand what it's in reference to. + +commit f19b052e0368ce8231687c6bfbd0479191fe6ce9 +Author: Zachary Tong +Date: Mon Apr 15 10:35:20 2019 -0400 + + Better error messages when pipelines reference incompatible aggs (#40068) + + Pipelines require single-valued agg or a numeric to be returned. + If they don't get that, they throw an exception. Unfortunately, this + exception text is very confusing to users because it usually arises + from pathing "through" multiple terms aggs. The final target is a numeric, + but it's the intermediary aggs that cause the problem. + + This commit adds the current agg name to the exception message + so the user knows which "level" is the issue. + +commit 2b539f83474cd3566d2b713102aa1cf6b88a3abf +Author: David Kyle +Date: Mon Apr 15 14:33:16 2019 +0100 + + [ML DataFrame] Data Frame stop all (#41156) + + Wild card support for the data frame stop API + +commit 3f00c29adb41e6f94a44da03607c0c3c93c8e6cb +Author: David Roberts +Date: Mon Apr 15 14:30:30 2019 +0100 + + [ML] Allow xpack.ml.max_machine_memory_percent higher than 100% (#41193) + + Values higher than 100% are now allowed to accommodate use + cases where swapping has been determined to be acceptable. + Anomaly detector jobs only use their full model memory + during background persistence, and this is deliberately + staggered, so with large numbers of jobs few will generally + be persisting state at the same time. Settings higher than + available memory are only recommended for OEM type + situations where a wrapper tightly controls the types of + jobs that can be created, and each job alone is considerably + smaller than what each node can handle. + +commit 23e40c040ae663892b4880372697d857ecc775a7 +Author: Guilherme Ferreira +Date: Mon Apr 15 15:10:57 2019 +0200 + + [Docs] Correct spelling of "_none_" (#41192) + +commit 9e94db1b6a49f3a22b372ad2c328ebab8b5133d8 +Author: Alpar Torok +Date: Mon Apr 15 15:24:01 2019 +0300 + + Mute failing test + + Tracked in #41172 + +commit 414debd740421a7d5b8ef9ab3e571e0f60c6ea71 +Author: Guilherme Ferreira +Date: Mon Apr 15 14:09:46 2019 +0200 + + [Docs] Correct spelling the "_none_" stopwords element (#41191) + +commit d30fec4914bcdab3a018106c22dc0c27679320e0 +Author: Jim Ferenczi +Date: Mon Apr 15 09:46:50 2019 +0200 + + Full text queries should not always ignore unmapped fields (#41062) + + Full text queries ignore unmapped fields since https://github.com/elastic/elasticsearch/issues/41022 + even if all fields in the query are unmapped. + This change makes sure that we ignore unmapped fields only if they are mixed + with mapped fields and returns a MatchNoDocsQuery otherwise. + + Closes #41022 + +commit fe9442b05b453de33f3054e80c5dd043036216ff +Author: Ioannis Kakavas +Date: Mon Apr 15 12:41:16 2019 +0300 + + Add an OpenID Connect authentication realm (#40674) (#41178) + + This commit adds an OpenID Connect authentication realm to + elasticsearch. Elasticsearch (with the assistance of kibana or + another web component) acts as an OpenID Connect Relying + Party and supports the Authorization Code Grant and Implicit + flows as described in http://ela.st/oidc-spec. It adds support + for consuming and verifying signed ID Tokens, both RP + initiated and 3rd party initiated Single Sign on and RP + initiated signle logout. + It also adds an OpenID Connect Provider in the idp-fixture to + be used for the associated integration tests. + + This is a backport of #40674 + +commit 2980a6c70fe4157f6bc11a8b00b6a2a8bb1f0535 +Author: Christoph Büscher +Date: Mon Apr 15 09:41:34 2019 +0200 + + Clarify some ToXContent implementations behaviour (#41000) + + This change adds either ToXContentObject or ToXContentFragment to classes + directly implementing ToXContent currently. This helps in reasoning about + whether those implementations output full xcontent object or just fragments. + + Relates to #16347 + +commit 5ef247dc91a42093e18d45e71fb8785da11f58a5 +Author: David Turner +Date: Sun Apr 14 10:39:50 2019 +0100 + + Further clarify cluster.initial_master_nodes (#41179) + + The following phrase causes confusion: + + > Alternatively the IP addresses or hostnames (if node name defaults to the + > host name) can be used. + + This change clarifies the conditions under which you can use a hostname, and + adds an anchor to the note introduced in (#41137) so we can link directly to it + in conversations with users. + +commit c8bc4ab0033f6fa485ec1e3b37b50c7029fbc148 +Author: Gordon Brown +Date: Fri Apr 12 16:53:50 2019 -0600 + + Improve Watcher test framework resiliency (#40658) + + It is possible for the watches tracked by ScheduleTriggerEngineMock to + get out of sync with the Watches in the ScheduleTriggerEngine + production code, which can lead to watches failing to run. + + This commit: + + 1. Changes TimeWarp to try to run the watch on all schedulers, rather than stopping after one which claims to have the watch registered. This reduces the impact of desynchronization between the mocking code and the backing production code. + 2. Makes ScheduleTriggerEngineMock respect pauses of execution again. This is necessary to prevent duplicate watch invocations due to the above change. + 3. Tweaks how watches are registered in ScheduleTriggerEngineMock to prevent race conditions due to concurrent modification. + 4. Tweaks WatcherConcreteIndexTests to use TimeWarp instead of waiting for watches to be triggered, as TimeWarp is more reliable and accomplishes the same goal. + +commit 47ba45732d173a5bb45309bf53e19ceaf44a35b1 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sat Apr 13 04:37:25 2019 +1000 + + Find and use non local IPv4 address while testing IP filtering (#40234) (#41141) + + For pattern "n:localhost" PatternRule#isLocalhost() matches + any local address, loopback address. + [Note: I think for "localhost" this should not consider IP address + as a match when they are bound to network interfaces. It should just + be loopback address check unless the intent is to match all local addresses. + This class is adopted from Netty3 and I am not sure if this is intended + behavior or maybe I am missing something] + + For now I have fixed this assuming the PatternRule#isLocalhost check is + correct by avoiding use of local address to check address denied. + + Closes #40194 + +commit e7375368d6e365343f8b21a09b6baef8ff61d304 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sat Apr 13 04:36:29 2019 +1000 + + Remove nested loop in IndicesStatsResponse (#40988) (#41138) + + This commit removes nested loop in `getIndices`. + +commit 7b190609ab59b9989cddeb8e14acfc4a22391a4e +Author: Lee Hinman +Date: Fri Apr 12 12:15:44 2019 -0600 + + (7.x) Use environment settings instead of state settings for Watcher config (#41158) + + Backport of (#41087) + + * Use environment settings instead of state settings for Watcher config + + Prior to this we used the settings from cluster state to see whether ILM was + enabled of disabled, however, these settings don't accurately reflect the + `xpack.ilm.enabled` setting in `elasticsearch.yml`. + + This commit changes to using the `Environment` settings, which correctly reflect + the ILM enabled setting. + + Resolves #41042 + +commit ef310886a7860fa5a2b246efff537d6fa1857756 +Author: Gordon Brown +Date: Fri Apr 12 10:43:04 2019 -0600 + + Add deprecation check for ILM poll interval <1s (#41096) + + ILM poll intervals of less than 1 second will not be allowed, so add a + deprecation check for that. + + Even though I'm pretty sure zero production clusters will do this, it's + best to be thorough. + +commit 9f3fae2c591b7a25e58847f86095c10ab589bf00 +Author: James Rodewig +Date: Fri Apr 12 12:26:39 2019 -0400 + + [DOCS] Fix code block length for Asciidoctor migration (#41152) + +commit dc1c5c264755a029cc60b4b56d2655c985ab27f6 +Author: James Rodewig +Date: Fri Apr 12 12:18:20 2019 -0400 + + [DOCS] Fix code block length for Asciidoctor migration (#41153) + +commit 62af000ff69fee8f35344916bda2d50d740f4b76 +Author: Nik Everett +Date: Fri Apr 12 11:31:08 2019 -0400 + + Docs: Fix rendering of APIs that mention version (#41154) + + Fixes rendering the `version_qualified` attribute in the docs. This + attribute includes `-SNAPSHOT` but does not include `-alpha` or `-beta` + and represents the `version` field as returned by `GET /` or + `GET /_cat/plugins`. Without this change we just drop the entire line on + the floor rather than render it so the output looks bad. + +commit 3be413627976908e8a01922d241a0d61620c8e65 +Author: James Rodewig +Date: Fri Apr 12 11:19:21 2019 -0400 + + [DOCS] Fix code block length for Asciidoctor migration (#41151) + +commit c379206c1e2bae2a88e6d3864617263f79f76d26 +Author: Nik Everett +Date: Fri Apr 12 10:11:14 2019 -0400 + + Fix some documentation urls in rest-api-spec (#40618) (#41145) + + Fixes some documentation urls in the rest-api-spec. Some of these URLs + pointed to 404s and a few others pointed to deprecated documentation + when we have better documentation now. I'm not consistent about `master` + vs `current` because we're not consistent in other places and I think we + should solve all of those at once with something a little more + automatic. + +commit b74d02944e1203d5817affe6826d37c62c264801 +Author: David Turner +Date: Fri Apr 12 10:45:09 2019 +0100 + + Clarify initial_master_nodes must match node.name (#41137) + + ... and emphasize that this includes any trailing qualifiers. + +commit ff64314b2b976aad411dede92ec8ffa247e5d5de +Author: Alpar Torok +Date: Fri Apr 12 12:38:29 2019 +0300 + + Simplify testclusters, don't allow cross project clusters (#40972) + + * Simplify testclusters, don't allow cross project clusters + +commit bee892006a5271c09f84ae1d606844e25cf9ad1d +Author: Alpar Torok +Date: Fri Apr 12 12:31:30 2019 +0300 + + Use the built image in docker tests (#40314) + + Instead of allowing docker-compose to rebuild it. + With this change we tag the image with a test label, and use that + in the testing as this is simpler that dealing with a dynamically + generated docker-compose file. + +commit d01c1f3ba0b93ca94b0d963cd3ce5dccfe3f15f3 +Author: Martijn van Groningen +Date: Fri Apr 12 11:14:19 2019 +0200 + + Added enrich policy definition. (#41003) + + Relates to #32789 + +commit 8af930c468b72c4e86b6594f12d5e6ee6b8200b0 +Author: Ignacio Vera +Date: Fri Apr 12 09:16:33 2019 +0200 + + Improve error message when polygons contains twice the same point in no-consecutive position (#41051) (#41133) + + When a polygon contains a self-intersection due to have twice the same point in no-consecutive position, the polygon builder tries to split the polygon. During the split one of the polygons become invalid as it is not closed and an error is thrown which is not related to the real issue. + + We detect this situation now and throw a more meaningful error. + +commit b66ad34565b0f5460c864babba9e344fbe9b84b6 +Merge: 5aecd9d0e22 3df6798c4cf +Author: Martijn van Groningen +Date: Fri Apr 12 08:24:59 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 3df6798c4cf8ecee24e6308f2d13754b62562970 +Author: Hendrik Muhs +Date: Fri Apr 12 07:29:55 2019 +0200 + + Rollup/DataFrame: disallow partial results (#41114) + + disallow partial results in rollup and data frame, after this change the client throws an error directly + replacing the previous runtime exception thrown, allowing better error handling in implementations. + +commit e9999dfa1d4adcb188a109c3db52cadacc254f63 +Author: Nhat Nguyen +Date: Thu Apr 11 22:18:31 2019 -0400 + + Init global checkpoint after copy commit in peer recovery (#40823) + + Today a new replica of a closed index does not have a safe commit + invariant when its engine is opened because we won't initialize the + global checkpoint on a recovering replica until the finalize step. With + this change, we can achieve that property by creating a new translog + with the global checkpoint from the primary at the end of phase 1. + +commit 79c7a57737c7e8201512bab4be8efb4d7a97841c +Author: Antonio Matarrese +Date: Thu Apr 11 17:38:25 2019 -0500 + + Use the breadth first collection mode for significant terms aggs. (#29042) + + This helps avoid memory issues when computing deep sub-aggregations. Because it + should be rare to use sub-aggregations with significant terms, we opted to always + choose breadth first as opposed to exposing a `collect_mode` option. + + Closes #28652. + +commit 0f496842fdbeb9a8d11aa858da3802e57430a52c +Author: Nhat Nguyen +Date: Thu Apr 11 18:41:21 2019 -0400 + + Fix msu assertion in restore shard history test + + Since #40249, we always reinitialize max_seq_no_of_updates to max_seq_no + when a promoting primary restores history regardless of whether it did + rollback previously or not. + + Closes #40929 + +commit 9c36ab4ab457115110745a7df1e2b458b6b976c5 +Author: Nhat Nguyen +Date: Thu Apr 11 15:53:02 2019 -0400 + + Adjust bwc version for flush parameter validation + + Relates to #40213 + +commit 5cdd87deb7490cc6598c6f1f0761c2ee1e889b99 +Author: Ryan Ernst +Date: Thu Apr 11 13:58:47 2019 -0700 + + Remove settings members from Node (#40811) + + This commit removes the settings member variable from Node. + This member made it confusing which settings should actually be looked + at. Now all settings are accessed through the final environment. + +commit b522de975da43133f1321df24dcecbd34feecd31 +Author: David Turner +Date: Thu Apr 11 20:26:18 2019 +0100 + + Move primary term from replicas proxy to repl op (#41119) + + A small refactoring that removes the primaryTerm field from ReplicasProxy and + instead passes it directly in to the methods that need it. Relates #40706. + +commit e120deb08ff8689a6cd8c0af97cb92d0c3a12128 +Author: Lisa Cawley +Date: Thu Apr 11 12:04:04 2019 -0700 + + [DOCS] Fixes callout for Asciidoctor migration (#41127) + +commit 05cf53934ad4ecff64f6e392a471578a41e2dd02 +Author: Benjamin Trent +Date: Thu Apr 11 13:54:41 2019 -0500 + + [ML] checking if p-tasks metadata is null before updating state (#41091) (#41123) + + * [ML] checking if p-tasks metadata is null before updating state + + * Adding test that validates fix + + * removing debug println + +commit 9e32e36799e4c770b5f96262601b6cf5680fcbdc +Author: Benjamin Trent +Date: Thu Apr 11 11:19:56 2019 -0500 + + [ML] fixing test related to #40963 (#41074) (#41116) + +commit 233df6b73b2ce5a063717fda8dc8a5f664e02247 +Author: Armin Braun +Date: Thu Apr 11 16:01:52 2019 +0200 + + Make Transport Shard Bulk Action Async (#39793) (#41112) + + This is a dependency of #39504 + + Motivation: + By refactoring `TransportShardBulkAction#shardOperationOnPrimary` to async, we enable using `DeterministicTaskQueue` based tests to run indexing operations. This was previously impossible since we were blocking on the `write` thread until the `update` thread finished the mapping update. + With this change, the mapping update will trigger a new task in the `write` queue instead. + This change significantly enhances the amount of coverage we get from `SnapshotResiliencyTests` (and other potential future tests) when it comes to tracking down concurrency issues with distributed state machines. + + The logical change is effectively all in `TransportShardBulkAction`, the rest of the changes is then simply mechanically moving the caller code and tests to being async and passing the `ActionListener` down. + + Since the move to async would've added more parameters to the `private static` steps in this logic, I decided to inline and dry up (between delete and update) the logic as much as I could instead of passing the listener + wait-consumer down through all of them. + +commit 93d6766afe948b1ce85e246169b46c4596b44ac7 +Author: James Rodewig +Date: Thu Apr 11 08:48:42 2019 -0400 + + [DOCS] Improve docs for 'elasticsearch-keystore add-file' command (#41084) + +commit 9cfe19413130cd75366f926f849d2dc658ad3dd2 +Author: Armin Braun +Date: Thu Apr 11 13:47:14 2019 +0200 + + Disable TestingConventions Check in Reindex Module (#41100) (#41110) + + * This is currently failing with: + + ``` + Test classes are not included in any enabled task (:modules:reindex:test): + * org.elasticsearch.client.documentation.ReindexDocumentationIT + * org.elasticsearch.index.reindex.ManyDocumentsIT + * org.elasticsearch.index.reindex.ReindexClientYamlTestSuiteIT + * org.elasticsearch.index.reindex.ReindexWithoutContentIT + * org.elasticsearch.index.reindex.remote.ReindexFromOldRemoteIT + ``` + + * Same fix as in #38546 + +commit a6dc20a09dd8f12b05dca680cff986622c15134f +Author: Mark Vieira +Date: Thu Apr 11 03:38:12 2019 -0700 + + Create heap dump on OOME (#41031) + +commit b967a97f8eb770217f7f2087678387267c1717bd +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Apr 11 12:09:53 2019 +0200 + + Reindex from remote deprecation warning (#41005) + + If a reindex from remote request contains an index name that is URL + escaped, we now issue a warning to be able to not support this in 8.0. + +commit 736c7285d4ca8bef1afe684924ca062e22c823bd +Author: Dimitris Athanasiou +Date: Thu Apr 11 11:29:51 2019 +0300 + + [ML] Fix scroll size comparison in DatafeedUpdate.isNoop (#41056) (#41079) + + Note this does not affect users as the method is only used in tests. + +commit 999462f46082c2ab851d5ea91c42e63622299e20 +Author: James Rodewig +Date: Wed Apr 10 14:46:51 2019 -0400 + + [DOCS] Document limits for JSON objects with `ignore_malformed` mapping setting (#40976) + +commit a0b54a9a92227e1919f2eeff252355b1311907fd +Author: James Rodewig +Date: Wed Apr 10 14:35:24 2019 -0400 + + [DOCS] Restructure `ids` to new query docs format (#41077) + +commit 24446ceae0932eb01f3784bd3842c859103e0b3c +Author: Jason Tedor +Date: Wed Apr 10 13:38:44 2019 -0400 + + Add packaging to cluster stats response (#41048) + + This commit adds a packaging_types field to the cluster stats response + that outlines the build flavors and types present in a cluster. + +commit 884c3fd7caaf38b33fe4458ba62c7d3f9247287c +Author: Ryan Ernst +Date: Wed Apr 10 10:43:01 2019 -0700 + + Use separate tasks per bwc artifact (#40703) (#40826) + + This commit changes the bwc builds from a single task for a branch to a + task for each bwc artifact. This reduces the bwc build time when only + needing a specific artifact, for example when running cluster restart + tests on a mac, the windows artifacts or rpm/debs are not needed. + +commit c37b127a073cfcd39bcff49c3156cc3be932c62b +Author: Hendrik Muhs +Date: Wed Apr 10 18:43:41 2019 +0200 + + fix a timing issue: isFinished is used for a busy loop in testing, (#41055) + + test: ensure state is persisted before the isFinished is changed + + fixes #41046 + +commit e611334b2bbf73e04ba63a1e5304cc7db02e0a28 +Author: Zachary Tong +Date: Wed Apr 10 11:32:53 2019 -0400 + + Add 7.0.1 version constant + +commit 1eff8976a8565b1781bd4c06cef0e3a2e09e1b09 +Author: Martijn van Groningen +Date: Wed Apr 10 14:06:06 2019 +0200 + + Deprecate AbstractHlrc* and AbstractHlrcStreamable* base test classes (#41014) + + * moved hlrc parsing tests from xpack to hlrc module and removed dependency on hlrc from xpack core + + * deprecated old base test class + + * added deprecated jdoc tag + + * split test between xpack-core part and hlrc part + + * added lang-mustache test dependency, this previously came in via + hlrc dependency. + + * added hlrc dependency on a qa module + + * duplicated ClusterPrivilegeName class in xpack-core, since x-pack + core no longer has a dependency on hlrc. + + * replace ClusterPrivilegeName usages with string literals + + * moved tests to dedicated to hlrc packages in order to remove Hlrc part from the name and make sure to use imports instead of full qualified class where possible + + * remove ESTestCase. from method invocation and use method directly, + because these tests indirectly extend from ESTestCase + +commit 64a05e522f1911aa8cb38a2f6c4fb018ff0850fa +Author: Chris Earle +Date: Wed Apr 10 10:09:28 2019 -0400 + + Properly handle Monitoring exporters all disabled (#40920) (#41043) + + When monitoring exporters are all disabled, which must be done + explicitly, _and_ monitoring collection is enabled, then + any call to `_xpack/monitoring/_bulk` will create a task that + never closes _and_ ES collection will stop happening because + a semaphore is never marked as completed. + + This also simplifies the async `ExportBulk` code by removing the + third step (second async step, `close`) entirely because it was + entirely unnecessary by both implementations. + +commit d00d3f4afa18b3e0dc7029328d292f5a84e1ab32 +Author: Christoph Büscher +Date: Wed Apr 10 15:43:04 2019 +0200 + + Mute DataFrameTransformCheckpointTests#testGetBehind + +commit 0a85d1fe03ead83e39208d1e27ab1a7ba6a550c0 +Author: Zachary Tong +Date: Wed Apr 10 09:41:04 2019 -0400 + + Add missing "long form" 7.0 breaking changes (#41049) + + These are breaking changes that were present in the release notes but the PR didn't include any "narrative long form" description for the docs. + +commit 9aa8ab58ee65989df54fb914b9ae3c5935aba3bf +Author: Zachary Tong +Date: Wed Apr 10 01:38:57 2019 -0400 + + Add 7.0.0 Release Notes (#41029) + +commit f5014ace64088b3e76cfeb6c06161af1c8fce2e9 +Author: Przemysław Witek +Date: Wed Apr 10 15:43:35 2019 +0200 + + [ML] Add validation that rejects duplicate detectors in PutJobAction (#40967) (#41072) + + * [ML] Add validation that rejects duplicate detectors in PutJobAction + + Closes #39704 + + * Add YML integration test for duplicate detectors fix. + + * Use "== false" comparison rather than "!" operator. + + * Refine error message to sound more natural. + + * Put job description in square brackets in the error message. + + * Use the new validation in ValidateJobConfigAction. + + * Exclude YML tests for new validation from permission tests. + +commit 799541e0681ef9b16ec6a9a4219b2ea573750abd +Author: Dimitrios Liappis +Date: Wed Apr 10 16:37:16 2019 +0300 + + Mute DateTimeUnitTests.testConversion (#40738) + + Due to #39617 + + Backport of #40086 + +commit 722362e402f991b3b813980fb4f36bd6c9b6b2ac +Author: Ed Savage +Date: Wed Apr 10 13:01:14 2019 +0100 + + Mute MachineLearningIt#testDeleteExpiredData + + Tracked in #41070 + +commit adf3393a4eb6b2d90bf298df1e0793795ea007c2 +Author: Albert Zaharovits +Date: Wed Apr 10 15:02:33 2019 +0300 + + Deprecate permission over aliases (#38059) (#41060) + + This PR generates deprecation log entries for each Role Descriptor, + used for building a Role, when the Role Descriptor grants more privileges + for an alias compared to an index that the alias points to. This is done in + preparation for the removal of the ability to define privileges over aliases. + There is one log entry for each "role descriptor name"-"alias name" pair. + On such a notice, the administrator is expected to modify the Role Descriptor + definition so that the name pattern for index names does not cover aliases. + + Caveats: + * Role Descriptors that are not used in any authorization process, + either because they are not mapped to any user or the user they are mapped to + is not used by clients, are not be checked. + * Role Descriptors are merged when building the effective Role that is used in + the authorization process. Therefore some Role Descriptors can overlap others, + so even if one matches aliases in a deprecated way, and it is reported as such, + it is not at risk from the breaking behavior in the current role mapping configuration + and index-alias configuration. It is still reported because it is a best practice to + change its definition, or remove offending aliases. + +commit 38f471ae1c2db227e93919ae1384e59f89167223 +Author: Costin Leau +Date: Wed Apr 10 13:14:18 2019 +0300 + + SQL: Change schema calls to empty set (#41034) + + As empty string has a certain meaning, the JDBC driver returns an empty + set instead for better client compatibility. + + Fix #41028 + + (cherry picked from commit 4cbafa585b7a514eb6c156606dd516324cd3980a) + +commit 4263a28039f73ee579da70bb095ed3b06b1b5138 +Author: Jim Ferenczi +Date: Wed Apr 10 11:37:37 2019 +0200 + + Fix rewrite of inner queries in DisMaxQueryBuilder (#40956) + + This commit implements missing rewrite for the DisMaxQueryBuilder. + + Closes #40953 + +commit ecbb588a140eb01a937e9a1dc59bf5f740c12da7 +Author: Adrien Grand +Date: Wed Apr 10 11:15:45 2019 +0200 + + Fix command line to run doc tests in docs/README. (#40959) + + It runs `gradle` instead of `gradlew` and has an extra `\`. + +commit 683cf56982a389d5599b1dcc425f18583d454cea +Author: Adrien Grand +Date: Wed Apr 10 11:33:49 2019 +0200 + + Update headline of the "removal of types" doc page to match changes in 7.0. (#40868) + + Currently it describes what broke in 6.0. + +commit f9018ab11bde75ad619122073b91408e15be1e1b +Author: Hendrik Muhs +Date: Wed Apr 10 08:00:58 2019 +0200 + + [ML-DataFrame] create checkpoints on every new run (#40725) + + Use the checkpoint service to create a checkpoint on every new run. Expose checkpoints stats on _stats endpoint. + +commit 46b0fdae33868bbeab142b1dc8d12402ba3308fd +Author: Martijn van Groningen +Date: Wed Apr 10 07:59:17 2019 +0200 + + Add realistic hlrc request serialization test base class and (#40362) + + changed hlrc ccr request tests to use AbstractRequestTestCase base class. + + This way the request classes are tested in a more realistic setting. + Note this change also adds a test dependency on xpack core module. + + Similar to #39844 but then for hlrc request serialization tests. + + Removed iterators from hlrc parsing tests. + Use empty xcontent registries. + + Relates to #39745 + +commit 0157ebf947df9f87e8b28378054da00d511fd11f +Author: Andrei Stefan +Date: Wed Apr 10 08:33:34 2019 +0300 + + Increase the interval filtering for CURRENT_DATE/TODAY tests (#40999) + + (cherry picked from commit a5911330aecee90da5401e468b9976f66a2556f0) + +commit bb6f060f744b8eb42b4cf44ae9454ad580b6a551 +Author: Jason Tedor +Date: Tue Apr 9 23:32:37 2019 -0400 + + Add log message to forget follower test + + This commit adds a log message to help debug failures in a forget + follower test. + +commit 3aae98f92213d0e098f90d0ffba4403e94258465 +Author: Jason Tedor +Date: Tue Apr 9 22:57:35 2019 -0400 + + Add debug logging for leases sync on recovery test + + This commit adds some debug logging for a retention leases sync on + recovery test. + +commit d38214060e204ac18dac556c71294386e05d9173 +Author: Julie Tibshirani +Date: Tue Apr 9 17:29:54 2019 -0700 + + Mute ClusterDisruptionIT#testCannotJoinIfMasterLostDataFolder. + + Tracked in #41047. + +commit c0f715b3374e9f22b61661d3af215c69b22fb069 +Author: Jason Tedor +Date: Tue Apr 9 20:21:34 2019 -0400 + + Fix Docker build when sourced from artifacts + + This commit fixes an issue when the artifact used to build the Docker + image is sourced from artifacts.elastic.co. In particular, the artifact + was not downloaded to the proper location. + +commit 0d5f86a001dc88e38a2b824da998472ecc627d34 +Author: Julie Tibshirani +Date: Tue Apr 9 17:09:08 2019 -0700 + + Mute RollupIndexerStateTests#testIndexing. + + Tracked in #41046. + +commit 0702c721514313fcac19438f2376e21e2faeb470 +Author: Julie Tibshirani +Date: Tue Apr 9 16:27:24 2019 -0700 + + Mute DataFrameGetAndGetStatsIT#testGetPersistedStatsWithoutTask. + + Tracked in #40963. + +commit 21c5d7e95fe0c4b4f45190ee7522cdfdb7f2df53 +Author: Julie Tibshirani +Date: Tue Apr 9 15:46:53 2019 -0700 + + Mute CcrRetentionLeaseIT#testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes. + + Tracked in #39331. + +commit 2e2e11f7b4db5bf01a606a5099fcad2621a07a78 +Author: James Rodewig +Date: Tue Apr 9 18:18:26 2019 -0400 + + [DOCS] Fix broken links for 7.0 release (#41036) + + * [DOCS] Remove references to deprecated 'zen.discovery.minimum_master_nodes' setting + +commit cbae617898b9ef041f9be53516e6b0aa0c788f65 +Author: Julie Tibshirani +Date: Tue Apr 9 14:48:10 2019 -0700 + + Mute IndexFollowingIT#testFollowIndex as we await a fix. + + Tracked in #41037. + +commit a417905098edc3a500b2c45b64a73993faea7620 +Author: Julie Tibshirani +Date: Tue Apr 9 13:34:58 2019 -0700 + + Mute RareClusterStateIT#testDelayedMappingPropagationOnPrimary as we await a fix. + + Tracked in #41030. + +commit a0fc2461d783863be2586ac64f775c7b85ef11e3 +Author: Julie Tibshirani +Date: Tue Apr 9 11:33:18 2019 -0700 + + Mute DedicatedClusterSnapshotRestoreIT#testSnapshotWithStuckNode as we await a fix. + +commit 1287c7d91f8610cd4f1b480e4a07f11649f7754a +Author: Mark Vieira +Date: Tue Apr 9 11:52:50 2019 -0700 + + [Backport] Replace usages RandomizedTestingTask with built-in Gradle Test (#40978) (#40993) + + * Replace usages RandomizedTestingTask with built-in Gradle Test (#40978) + + This commit replaces the existing RandomizedTestingTask and supporting code with Gradle's built-in JUnit support via the Test task type. Additionally, the previous workaround to disable all tasks named "test" and create new unit testing tasks named "unitTest" has been removed such that the "test" task now runs unit tests as per the normal Gradle Java plugin conventions. + + (cherry picked from commit 323f312bbc829a63056a79ebe45adced5099f6e6) + + * Fix forking JVM runner + + * Don't bump shadow plugin version + +commit 321f93c4f95255536ae85cd92f9a15a79e48674e +Author: Jason Tedor +Date: Tue Apr 9 14:24:54 2019 -0400 + + Wait for all listeners in checkpoint listeners test + + It could be that we try to shutdown the executor pool before all the + listeners have been invoked. It can happen that one was not invoked if + it timed out and was in the process of being notified that it timed out + on the executor. If we do this shutdown then, a listener will be met + with rejected execution exception. To address this, we first wait until + all listeners have been notified (or timed out) before proceeding with + shutting down the executor. + + Relates #40970 + +commit ebba9393c1abca0305ecbb2d883b2d2b1a56acc8 +Author: Jason Tedor +Date: Tue Apr 9 13:50:47 2019 -0400 + + Fix unsafe publication of invalid license enforcer (#40985) + + The invalid license enforced is exposed to the cluster state update + thread (via the license state listener) before the constructor has + finished. This violates the JLS for safe publication of an object, and + means there is a concurrency bug lurking here. This commit addresses + this by avoiding publication of the invalid license enforcer before the + constructor has returned. + +commit e71db0531eb763ce91a82e8c7f1820719cc7d86c +Author: Armin Braun +Date: Tue Apr 9 19:40:01 2019 +0200 + + Fix Race in AsyncTwoPhaseIndexerTests.testStateMachine (#40947) (#41013) + + * The step is incremented by the listner in `org.elasticsearch.xpack.core.indexing.AsyncTwoPhaseIndexerTests.MockIndexer#onFinish` after isFinished is set to true, but the test only waited for `isFinished`, + fixed by calling `isFinished` last + * Also made `step` volatile since we are reading it from different thread from the one incrementing it + * Closes #40946 + +commit fdc1bdd4d3f33dc07008dcd15d8e9d806a1865a4 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Tue Apr 9 16:47:21 2019 +0100 + + [ML][TEST] Fix randomly failing HLRC test (#40973) + + Made changes to ensure that unique IDs are generated for model snapshots + used by the deleteExpiredDataTest test in the MachineLearningIT suite. + + Previously a sleep of 1s was performed between jobs under the assumption + that this would be sufficient to guarantee that the timestamps used in + the composition of the snapshot IDs would be different. + + The new approach is to wait on the condition that the old and new + timestamps are in fact different (to 1s resolution). + +commit 2ac514b909a28963ac2681a57976343cd3180a8f +Author: Costin Leau +Date: Tue Apr 9 18:38:54 2019 +0300 + + SQL: Fix catalog filtering in SYS COLUMNS (#40583) + + Properly treat '%' as a wildcard for catalog filtering instead of doing + a straight string match. + Table filtering now considers aliases as well. + Add escaping char for LIKE queries with user defined params + Fix monotony of ORDINAL_POSITION + Add integration test for SYS COLUMNS - currently running only inside + single_node since the cluster name is test dependent. + Add pattern unescaping for index names + + Fix #40582 + + (cherry picked from commit 8e61b77d3f849661b7175544f471119042fe9551) + +commit 3b9ab5da04e3131267cc14c1a80b0a7915c2f633 +Author: Alexander Reelsen +Date: Tue Apr 9 16:35:25 2019 +0200 + + Fix order of request body search parameter names in documentation (#40777) + + The order was random, which made it super hard to find anything. This + changes the order to be alphabetically. + +commit 040a4961c783318803c8dc95970ba53484cdc8bb +Author: Martijn van Groningen +Date: Tue Apr 9 09:46:05 2019 +0200 + + Revert "Revert "Change HLRC CCR response tests to use AbstractResponseTestCase base class. (#40257)"" (#40971) + + This reverts commit df91237a94fd3d3ae954eb1845c434dda692d087. + +commit c5a77e5d8cbeed241e5bbc403f0b3680d4e22173 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Apr 9 15:02:03 2019 +0200 + + Node repurpose tool docs (#40525) + + Added documentation for node repurpose tool and included documentation on how to repurpose nodes safely. Adjusted order of tools in `elasticsearch-node` tool since the repurpose tool is most likely to be used. + + Co-Authored-By: David Turner + +commit 40638d7b28dc1714af737ac603f667cab77be46e +Author: Christoph Büscher +Date: Tue Apr 9 13:46:33 2019 +0200 + + [Docs] Delete explanation for completion suggester default analyzer choice (#36720) + + The explanation given in the completion suggester documentation why we use the + "simple" analyzer as the default is no longer valid. Since we still use "simple" + as the default, we should just delete the explanation that doesn't fit anymore. + + Closes #36715 + +commit 5aecd9d0e221822e4e75f8cd3d5c1c4d800c28fe +Author: Martijn van Groningen +Date: Tue Apr 9 10:07:10 2019 +0200 + + Fixed build issue after merge + +commit 5a1d5cca4f865acc8b2d2f92ff96923274978949 +Merge: e6bdfea4745 64c98c632a2 +Author: Martijn van Groningen +Date: Tue Apr 9 09:59:24 2019 +0200 + + Merge remote-tracking branch 'es/7.x' into enrich-7.x + +commit 08ecdfe20ecfa58631aca8ac0bc49470f9a8ea94 +Author: David Turner +Date: Tue Apr 9 07:21:51 2019 +0100 + + Short-circuit rebalancing when disabled (#40966) + + Today if `cluster.routing.rebalance.enable: none` then rebalancing is disabled, + but we still execute `balanceByWeights()` and perform some rather expensive + calculations before discovering that we cannot rebalance any shards. In a large + cluster this can make cluster state updates occur rather slowly. With this + change we check earlier whether rebalancing is globally disabled and, if so, + avoid the rebalancing process entirely. + + Relates #40942 which was reverted because of egregiously faulty tests. + +commit 64c98c632a20902eeeeb4e36c762954b66b81cfd +Author: Mark Vieira +Date: Mon Apr 8 21:17:49 2019 -0700 + + Ignore failing tests (#40994) + +commit 69421612e53295603c9ecd1589e134e4b6e18b8f +Author: Nhat Nguyen +Date: Mon Apr 8 22:00:48 2019 -0400 + + Mute testRecoverMissingAnalyzer + + Tracked at #40867 + +commit 713e5c987b72f0b94944caa9000e6879bc82374f +Author: Nhat Nguyen +Date: Mon Apr 8 15:42:48 2019 -0400 + + Adjust init map size of user data of index commit (#40965) + + The number of user data attributes of an index commit has increased + from 6 to 8, but we forgot to adjust. This change increases the initial + size of that map to avoid resizing. + +commit a9416708d00762862636bf3b8e4a117cae4eb357 +Author: Nhat Nguyen +Date: Sun Apr 7 13:32:54 2019 -0400 + + Ensure relocation occur in testRelocationWithConcurrentIndexing (#40801) + + If the relocation is throttled, the subsequent search request on the + target node (i.e., with preference _only_nodes=target_node) will fail + because some shards have not moved to that node yet. With this change, + we will wait for the relocation happens by busily checking the routing + table of the testing index on the target node. + + Closes #34950 + +commit 1ed4f215acee3ce936c6d9ae4179f932f74679af +Author: Lisa Cawley +Date: Mon Apr 8 18:54:29 2019 -0700 + + [DOCS] Add notable-breaking-changes tags (#40991) + +commit 9e271b9ec2e7985e82496912875022afa380d22a +Author: James Rodewig +Date: Mon Apr 8 16:11:32 2019 -0400 + + [DOCS] Fix broken link to Elasticsearh Docker source code (#40979) + + * [DOCS] Fix broken link to Elasticsearch Docker source code + + * [DOCS] Link to Dockerfile in elastic/elasticsearch repo + + * [DOCS] Link to Docker source files in elastic/elasticsearch repo + +commit 26d8ecfe07d9d4254e12185d81c2f7a8f5869cef +Author: Jason Tedor +Date: Mon Apr 8 16:10:24 2019 -0400 + + Fix unsafe publication in opt-out query cache (#40957) + + This opt-out query cache has an unsafe publication issue, where the + cache is exposed to another thread (namely the cluster state update + thread) before the constructor has finished execution. This exposes the + opt-out query cache to concurrency bugs. This commit addresses this by + ensuring that the opt-out query cache is not registered as a listener + for license state changes until after the constructor has returned. + +commit 335955b8742ee256914193092cfdc6687afd19c0 +Author: Christoph Büscher +Date: Mon Apr 8 20:47:53 2019 +0200 + + Some internal refactorings in AnalysisRegistry (#40609) + + Reducing some methods scope and marking them as static where possible. Removing + "alias" support from AnalysisRegistry#produceAnalyze and changing that method to + return a NamedAnalyzer instead of having a side effect on the analyzer map passed in. + Also, CustomAnalyzerProvider doesn't seem to need the `environment` field. + +commit efe45011a57ed36cf8405af1cb6d5e1b23918645 +Author: Lisa Cawley +Date: Mon Apr 8 11:36:29 2019 -0700 + + [DOCS] Fix deprecation notification (#40980) + +commit 5990c54801e7508aca42844ac69b71f1055784cb +Author: Shaunak Kashyap +Date: Mon Apr 8 09:31:26 2019 -0700 + + [7.x] Docs: Simplifying setup by using module configuration variant syntax (#40879) (#40974) + + Backports the following commits to 7.x: + - Docs: Simplifying setup by using module configuration variant syntax (#40879) + +commit 2206491277c44fc7f3b0ba42779c999627dfdc19 +Author: Marios Trivyzas +Date: Mon Apr 8 16:43:36 2019 +0200 + + SQL: Refactor args verification of In & conditionals (#40916) + + Move verification of arguments for Conditional functions and IN + from `Verifier` to the `resolveType()` method of the functions. + + (cherry picked from commit 241644aac57baee1eb128b993ee410c7d08172a5) + +commit 8eef92fafd8944c95f526ce63d4ebfbc4992e6f4 +Author: David Turner +Date: Mon Apr 8 15:58:56 2019 +0100 + + Revert "Short-circuit rebalancing when disabled (#40942)" + + This reverts commit f78e6ef73b4ac47f223ee471336a011d3deced0a. + +commit 2569fb60de676511be7665f24102736221056409 +Author: Mark Vieira +Date: Mon Apr 8 07:05:12 2019 -0700 + + Avoid sharing source directories as it breaks intellij (#40877) + + * Avoid sharing source directories as it breaks intellij + * Subprojects share main project output classes directory + * Fix jar hell + * Fix sql security with ssl integ tests + * Relax dependency ordering rule so we don't explode on cycles + +commit 21b99a3aebaa9cbed96f62f773ef71137e479a54 +Author: Alpar Torok +Date: Fri Apr 5 14:10:22 2019 +0300 + + Remove unneded cluster config from test (#40856) + + This configuration doesn't influence the logger test. + Should be removed to avoid confusion + +commit f78e6ef73b4ac47f223ee471336a011d3deced0a +Author: David Turner +Date: Mon Apr 8 14:57:42 2019 +0100 + + Short-circuit rebalancing when disabled (#40942) + + Today if `cluster.routing.rebalance.enable: none` then rebalancing is disabled, + but we still execute `balanceByWeights()` and perform some rather expensive + calculations before discovering that we cannot rebalance any shards. In a large + cluster this can make cluster state updates occur rather slowly. With this + change we check earlier whether rebalancing is globally disabled and, if so, + avoid the rebalancing process entirely. + +commit af874635baf2db891c24b1fba790708937f2bc01 +Author: Costin Leau +Date: Mon Apr 8 16:45:25 2019 +0300 + + SQL: Prefer resultSets over exceptions in metadata (#40641) + + Changed the JDBC metadata to return empty results sets instead of + throwing SQLFeatureNotSupported as it seems a more safer/compatible + approach for consumers. + + Fix #40533 + + (cherry picked from commit ef2d2527c2b5140556fd477e7ff6ea36966684da) + +commit 3e078b2026f0736ef7595786008cee33fd9c0967 +Author: Jason Tedor +Date: Mon Apr 8 08:15:05 2019 -0400 + + Use bundled Java for all eligible versions in tests (#40928) + + This commit sets the version to ensure that we use the bundled Java when + running integration tests for all eligible versions. In particular, + since we started bundling Java with 7.0.0, this commits sets said + version to 7.0.0. + +commit bc0fe7d64d0de8544600b669fcbbe6afd16513bc +Author: Jim Ferenczi +Date: Mon Apr 8 11:22:38 2019 +0200 + + Handle min_doc_freq in phrase suggester (#40840) + + The phrase suggesters have an option to remove terms that have + a frequency lower than a provided min_doc_freq. However this value is + overwritten by the frequency of the original term in the popular mode. + This change ensures that we keep the maximum value between the provided + min_doc_value and the original term frequency as a threshold to select + candidates. + + Fixes #16764 + +commit fb5d7cf237bdccfba6a00f6b3128772bdd3f4134 +Author: David Kyle +Date: Mon Apr 8 11:06:24 2019 +0100 + + Mute AsyncTwoPhaseIndexerTests.testStateMachine + +commit c9ff630a12ab964e6dcb8c0b85e4167d14dcf324 +Author: Ioannis Kakavas +Date: Mon Apr 8 13:10:53 2019 +0300 + + Mute reindex integTest in FIPS (#40941) + + Relates: #40904 + +commit 84a410d5a89d7ddaaf0ea9f4ecf66a79ddd8421c +Author: Martijn van Groningen +Date: Mon Apr 8 11:24:06 2019 +0200 + + Revert "Change HLRC CCR response tests to use AbstractResponseTestCase base class. (#40257)" + + This reverts commit c29027d99e969d464fcb461921ae5b0de6f7d086. + +commit ddf17dfb1e7699c90006fa2c8665334a2e1a4e76 +Author: Marios Trivyzas +Date: Mon Apr 8 10:13:15 2019 +0200 + + SQL: Small code improvements of Pipes & Processors (#40909) + + - Remove superfluous methods that are already + defined in superclasses. + - Improve tests for null folding on conditionals + + (cherry picked from commit 67f9404f5004362e569353d1e950ffe5d7a9ab6e) + +commit c29027d99e969d464fcb461921ae5b0de6f7d086 +Author: Martijn van Groningen +Date: Mon Apr 8 07:08:36 2019 +0200 + + Change HLRC CCR response tests to use AbstractResponseTestCase base class. (#40257) + + This way the response classes are tested in a more realistic setting. + + Relates to #39745 + +commit 26c63e011502c929c32939b099a968c47d37416b +Author: Tim Vernum +Date: Mon Apr 8 13:23:12 2019 +1000 + + Add test for HTTP and Transport TLS on basic license (#40932) + + This adds a new security/qa test for TLS on a basic license. + + It starts a 2 node cluster with a basic license, and TLS enabled + on both HTTP and Transport, and verifies the license type, x-pack + SSL usage and SSL certificates API. + + It also upgrades the cluster to a trial license and performs that + same set of checks (to ensure that clusters with basic license + and TLS enabled can be upgraded to a higher feature license) + + Backport of: #40714 + +commit d5fcbf2f4a824a0c221760ebb655b67fe38ad679 +Author: Hendrik Muhs +Date: Sun Apr 7 21:43:35 2019 +0200 + + refactor onStart and onFinish to take runnables and executed them guarded by state (#40855) + + refactor onStart and onFinish to take action listeners and execute them when indexer is in indexing state. + +commit 4163e5976895e4a6d1f31de031ca8cbe367efd04 +Author: Jason Tedor +Date: Sun Apr 7 10:16:35 2019 -0400 + + Mute failing IndexShard local history test + + This test fails reliably with, so this commit mutes that test until a + fix is available. + +commit 690039914412e8c04ea86d775554267635c15bb5 +Author: Jason Tedor +Date: Sat Apr 6 17:23:51 2019 -0400 + + Be lenient when parsing build flavor and type on the wire (#40734) + + Today we are strict when parsing build flavor and types off the + wire. This means that if a later version introduces a new build flavor + or type, an older version would not be able to parse what that new + version is sending. For a practical example of this, we recently added + the build type "docker", and this means that in a rolling upgrade + scenario older nodes would not be able to understand the build type that + the newer node is sending. This breaks clusters and is bad. We do not + normally think of adding a new enumeration value as being a + serialization breaking change, it is just not a lesson that we have + learned before. We should be lenient here though, so that we can add + future changes without running the risk of breaking ourselves + horribly. It is either that, or we have super-strict testing + infrastructure here yet still I fear the possibility of mistakes. This + commit changes the parsing of build flavor and build type so that we are + still strict at startup, yet we are lenient with values coming across + the wire. This will help avoid us breaking rolling upgrades, or clients + that are on an older version. + +commit e44e84ab42e1daa8ca0a31997e73a9554dc317e7 +Author: Jason Tedor +Date: Sat Apr 6 10:17:29 2019 -0400 + + Suppress lease background sync failures if stopping (#40902) + + If the transport service is stopped, likely because we are shutting + down, and a retention lease background sync fires the logs will display + a warn message and stacktrace. Yet, this situaton is harmless and can + happen as a normal course of business when shutting down. This commit + suppresses the log messages in this case. + +commit a69ff8221f730a8d7a791c5136fbbea081a3ffd7 +Author: debadair +Date: Fri Apr 5 16:38:31 2019 -0700 + + [DOCS] Added settings page for ILM. (#40880) + + * [DOCS] Added settings page for ILM. + + * [DOCS] Adding ILM settings file + + * [DOCS] Moved the ILM settings to a separate section + + * [DOCS] Linked to the rollover docs. + + * [DOCS] Tweaked the "required" wording. + +commit b0ce3e0a10559b2e29ebfcf122db90f223d10c32 +Author: DeDe Morton +Date: Fri Apr 5 15:31:23 2019 -0700 + + [Docs] Remove extraneous text (#40914) + + Removes text that was likely introduced by copy/paste error. + +commit fd51780de28205b3a77d29dcd26f233e3de9ba2b +Author: Jack Conradson +Date: Fri Apr 5 15:15:14 2019 -0700 + + Move test classes to test root in Painless (#40873) + + This moves several test classes that were part of the main root to the test root. + These were part of the main root due to limitations prior to whitelist + customization. Without whitelist customization these can be moved to a test + context and removed from the base whitelists as they should not be user + facing. + +commit ac58b9bdedaaf8bf2b86dc4240bab75b4c2d9be2 +Author: Jason Tedor +Date: Fri Apr 5 17:45:30 2019 -0400 + + Fix date index name processor default date_formats (#40915) + + This commit is a correction of a doc bug in the docs for the ingest + date-index-name processor. The correct pattern is + yyyy-MM-dd'T'HH:mm:ss.SSSXX. This is due to the transition from Joda + time to Java time where Z does not mean the same thing between the two. + +commit aea4e6596f9be4b8849dd7b6f16ff99157110997 +Author: Ioannis Kakavas +Date: Sat Apr 6 00:42:14 2019 +0300 + + Source additional files correctly in elasticsearch-cli (#40890) + + Since we only source additional sources from the same dir as our + cli scripts, resolve the path relevant to $ES_HOME + +commit f4348843ba79f946ffb471b7762c6fe6baa6aa56 +Author: lcawl +Date: Fri Apr 5 14:26:22 2019 -0700 + + [DOCS] Adds placeholder for 7.0.0 release notes + +commit 2fd1689341aae3a1e7169d90f6e25dc1631933bb +Author: Jason Tedor +Date: Fri Apr 5 16:39:44 2019 -0400 + + Allow AVX-512 on JDK 11+ (#40828) + + We previously found a bug in the JVM where AVX-512 instructions could + crash the JVM to crash with a segmentation fault. This bug impacted JDK + 9 and JDK 10, but was most prominent on JDK 10 because AVX-512 was + enabled there by default. In JDK 11, this bug is reported fixed so this + commit restricts the disabling of AVX-512 to JDK 10 only. Since we no + longer support JDK 10 for any versions that this commit will be + integrated into (7.1, 8.0), we simply remove the disabling of this flag + from the JVM options. + +commit f34663282c694b8dffa3a9c5899288444058f28a +Author: Jay Modi +Date: Fri Apr 5 12:06:06 2019 -0600 + + Update apache httpclient to version 4.5.8 (#40875) + + This change updates our version of httpclient to version 4.5.8, which + contains the fix for HTTPCLIENT-1968, which is a bug where the client + started re-writing paths that contained encoded reserved characters + with their unreserved form. + +commit 360d8d40be8ac5d125ea34688c28d08c179ecd61 +Author: DeDe Morton +Date: Fri Apr 5 11:55:21 2019 -0700 + + [Docs] Change example to show col headers (#40822) + + Command needs `?v` so user can see the column headers. Otherwise the instructions in the note about checking the init and relo columns don't make sense + +commit 2ff19bc1b7adc2e886ebf88f18f227f7e0e33562 +Author: David Turner +Date: Fri Apr 5 19:10:10 2019 +0100 + + Use Writeable for TransportReplAction derivatives (#40905) + + Relates #34389, backport of #40894. + +commit a8dbb07546034313d155618055cd8c8728bd5a6a +Author: Benjamin Trent +Date: Fri Apr 5 11:34:20 2019 -0500 + + [ML] Changes default destination index field mapping and adds scripted_metric agg (#40750) (#40846) + + * [ML] Allowing destination index mappings to have dynamic types, adds script_metric agg + + * Making dynamic|source mapping explicit + +commit 4452e8e10f716931eab3028ca2973610318904ae +Author: Colin Goodheart-Smithe +Date: Thu Apr 4 18:16:11 2019 +0100 + + Mutes GatewayIndexStateIT.testRecoverBrokenIndexMetadata + +commit eb656244bf4ecd195a4e7083f58335a3e6c4bed8 +Author: Brandon Kobel +Date: Fri Apr 5 07:43:20 2019 -0700 + + Add Kibana application privileges for monitoring and ml reserved roles (#40651) + + * Add Kibana application privileges for monitoring and ml reserved roles + + * Adding test for kibana-.kibana application explicitly + + * Whoa there, fat fingered kibana and application... + + * And I copied something from monitoring I shouldn't have... + + * And actually doing what Yogesh recommended... + +commit 665f0d81aa5090374a7f51d557885d4f792ccbbf +Author: Benjamin Trent +Date: Fri Apr 5 09:01:01 2019 -0500 + + [ML] refactoring start task a bit, removing unused code (#40798) (#40845) + +commit 922a70ce326c8d6b98aedc366559e7e746f46f6f +Author: David Turner +Date: Fri Apr 5 09:21:23 2019 +0100 + + Remove unused import + + Relates #40863 + +commit d8956d260149708b81e81c2446f348090451d3ff +Author: David Turner +Date: Fri Apr 5 08:50:44 2019 +0100 + + Remove test-only customisation from TransReplAct (#40863) + + The `getIndexShard()` and `sendReplicaRequest()` methods in + TransportReplicationAction are effectively only used to customise some + behaviour in tests. However there are other ways to do this that do not cause + such an obstacle to separating the TransportReplicationAction into its two + halves (see #40706). + + This commit removes these customisation points and injects the test-only + behaviour using other techniques. + +commit 669d72e47a84deecc9b547e9ad753b5c269ab8c8 +Author: Alexander Reelsen +Date: Fri Apr 5 09:35:10 2019 +0200 + + Fix dense/sparse vector limit documentation (#40852) + + The documentation stated a wrong limit of dense/sparse vector sizes. + This was changed in #40597 but the documentation was not fixed. + +commit e6bdfea47454b22d11670e84bc9bace3fe21e599 +Author: Martijn van Groningen +Date: Wed Apr 3 13:19:29 2019 +0200 + + first commit + +commit 920024e06dedbc019bdf5fefbbe80852630005a9 +Author: Tim Vernum +Date: Fri Apr 5 18:28:09 2019 +1100 + + Remove dynamic objects from security index (#40715) + + The security index had a few "object" types with + + "dynamic": true + + However, this automatically creates a mapping for each field that is + created within those objects. This means that types are dynamically + inferred and "locked in" for future updates. + + Instead we want "dynamic": false which will allow us to store a range + of fields in these nested objects and retrieve them from the source, + without creating mapping types for those fields. + + Backport of: #40499 + +commit 809a5f13a43c465b3b986539d435a305a4ef7f5f +Author: Martijn van Groningen +Date: Fri Apr 5 08:01:56 2019 +0200 + + Make -try xlint warning disabled by default. (#40833) + + Many gradle projects specifically use the -try exclude flag, because + there are many cases where auto-closeable resource ignore is never + referenced in body of corresponding try statement. Suppressing this + warning specifically in each case that it happens using + `@SuppressWarnings("try")` would be very verbose. + + This change removes `-try` from any gradle project and adds it to the + build plugin. Also this change removes exclude flags from gradle projects + that is already specified in build plugin (for example -deprecation). + + Relates to #40366 + +commit 5a2eb07c0e99f4d0d254e1cd4bee3d72e2be9355 +Author: Nhat Nguyen +Date: Wed Apr 3 22:17:31 2019 -0400 + + Primary replica resync should not send ops without seqno (#40433) + + Primary-replica resync in a mixed-cluster between 6.x and 5.6 can send + operations without sequence number to a replica which already processed + operations with sequence number. This leads to the failure of that + replica for we trip the sequence number assertion when writing resync + operations without sequence number to translog. + +commit 8f72d72c4c2e4ddc18b3fa9d294c11c7586c6985 +Author: Deb Adair +Date: Tue Apr 2 16:01:51 2019 -0700 + + [DOCS] Removed redundant (not quite right) information about upgrades. + +commit a15f55b2de442a93e7d557a0280b768afec3ec84 +Author: Jim Ferenczi +Date: Thu Apr 4 21:16:14 2019 +0200 + + Rollup ignores time_zone on date histogram (#40844) + + When translating the original aggregation for the rollup indices, + the timezone of the date histogram is validated against the rollup + job but the value is not copied in the newly created date_histogram. + +commit 075078e7e05125f5e5de9e0e54c51d50998f58fe +Author: roy +Date: Fri Apr 5 01:58:41 2019 +0800 + + HLRC: fix uri encode bug when url path starts with '/' (#34436) + + This commit sets the authority of a URI to blank such that it does not + misinterpret slashes in the path as the authority. + + Closes #34433 + +commit 5347dec55e964d6aa77cc5816d5f7c5fb1a879a8 +Author: Gordon Brown +Date: Thu Apr 4 11:46:21 2019 -0600 + + Allow ILM to stop if indices have nonexistent policies (#40820) + + Prior to this PR, there is a bug in ILM which does not allow ILM to stop + if one or more indices have an index.lifecycle.name which refers to + a policy that does not exist - the operation_mode will be stuck as + STOPPING until either the policy is created or the nonexistent + policy is removed from those indices. + + This change allows ILM to stop in this case and makes the logging more + clear as to why ILM is not stopping. + +commit 65d25186d3438b0d52b2bcd73a9beef10bca6bb5 +Author: Przemyslaw Gomulka +Date: Thu Apr 4 18:42:05 2019 +0200 + + Bat scripts to work with JAVA_HOME with parentheses Backports (#40832) + + backports : + Bat scripts to work with JAVA_HOME with parentheses (#39712) + Link to SYSTEM_JAVA_HOME on windows (#40806) + +commit fb5a0652a8df09d92982bcadb6b4a4c532384102 +Author: Michael Basnight +Date: Thu Apr 4 09:49:12 2019 -0500 + + HLRC: Convert xpack methods to client side objects (#40705) + + This commit fixes a problem with BWC that was brought up in #40511. A + newer version of the code was emitting a new value for an enum to an + older version, and the older version could not handle that. It caused + the response to error. The MainResponse is now relaxed, and will accept + whatever values the server expose, and holds most of them as Strings + instead of complex objects. + + Fixes #40511 + +commit 9e8499d20b00670fd30c39a3acd73aa59ce41fff +Author: Nik Everett +Date: Thu Apr 4 12:02:55 2019 -0400 + + Docs: Pin two IDs in the rest client (#40785) + + We generate two pages with "funny" names: + * _changing_the_client_8217_s_initialization_code.html + * _changing_the_application_8217_s_code.html + + The leading `_` comes from us not specifying the name of the page. The + `8217` comes about because of the single quote character. This is a + funny name, but it is the name that we have so we shouldn't change it + without putting in a redirect. + + We're looking at switching these docs from being built with the + no-longer-maintained AsciiDoc project to being built with the + actively-maintained Asciidoctor project. Asciidoctor Doesn't include the + `8217`s in the generated ids. That is *better*, but we don't really want + to change the pages. Ultimately we'd prefer none of our pages start with + `_`, but that is a problem for a different time. + + Anyway, this pins the ids to their "funny" id so it won't change when we + switch to Asciidoctor. We'll remove it later, when we have more fine + control of our redirects. + +commit b7497bf2dff03cdb87e6bfba1b32355c4b06175d +Author: James Rodewig +Date: Thu Apr 4 11:37:08 2019 -0400 + + [DOCS] Remind users to include @ symbol when applying license file (#40688) + +commit 402f312c5e8923e64dff8dd315564e89ec3676b0 +Author: Colin Goodheart-Smithe +Date: Thu Apr 4 12:30:06 2019 +0100 + + Adds version 6.7.2 + +commit 38f4dda2bedc5f12e75bb29c0657cf8b364fe73e +Author: Taylor Scott +Date: Thu Apr 4 08:30:37 2019 -0700 + + Fix grammatical error in analysis.asciidoc (#40827) + +commit a042b22d248a74b80f726b61b820d8d536b28f08 +Author: Jack Conradson +Date: Wed Apr 3 09:46:33 2019 -0700 + + Move PainlessExecuteAction into action package (#40752) + +commit 7c27e5f243098d4251b865a7ac89636d874491f9 +Author: Adrien Grand +Date: Thu Apr 4 13:59:07 2019 +0200 + + Revert "Mute failing test" + + This reverts commit 1af2b2bfe65e346e332fe040ecdd9df54a81eeb5. + +commit fce1bbc20e9bc9ba5197ff90f5377a2293c64db1 +Author: Nik Everett +Date: Thu Apr 4 08:41:30 2019 -0400 + + Docs: Drop inline callouts from painless book (#40805) + + Drops the inline callouts from the painless reference book. These + callouts are incompatible with Asciidoctor and we'd very much like to + switch to Asciidoctor for building this book, partially because + Asciidoctor is actively developed and AsciiDoc is not, and partially + because it builds the book three times faster. + +commit 2756a3936be4122b6c480bdd647e9eca5d7f4bdb +Author: Nhat Nguyen +Date: Tue Mar 19 21:23:25 2019 -0400 + + Reject illegal flush parameters (#40213) + + This change rejects an illegal combination of flush parameters where + force is true, but wait_if_ongoing is false. This combination is trappy + and should be forbidden. + + Closes #36342 + +commit c4960ad73603f857a0e85cb03f09cb5a1229e238 +Author: Nhat Nguyen +Date: Tue Mar 19 21:27:08 2019 -0400 + + Ensure flush happen before closing an index (#40184) + + If there's an ongoing flush triggered by the translog flush threshold, + we may fail to execute a flush because waitIfOngoing is false by + default. + + Relates to #36342 + +commit e716b9ceee4745fcecf41b3adcfc540d2b7b3f90 +Author: Nhat Nguyen +Date: Wed Mar 27 01:23:35 2019 -0400 + + Ensure no scheduled refresh in testPendingRefreshWithIntervalChange + + If a refresh, which is scheduled by the setting change, executes after + the index-2 operation and win the refresh race (i.e., maybeRefresh) with + the scheduledRefresh that we are going to check, then the latter will + return false. + + Closes #39565 + Relates #39462 + + PR #40387 + +commit 11f2da9d5eec3fda1b91d544eca8358ca966e8ee +Author: James Rodewig +Date: Thu Apr 4 08:56:36 2019 -0400 + + [DOCS] Rewrite query def (#40746) + +commit 7c669026ac1ede18d62b7528125de3c02c6097e6 +Author: James Rodewig +Date: Thu Apr 4 08:48:37 2019 -0400 + + [DOCS] Document index.load_fixed_bitset_filters_eagerly (#40780) (#40842) + +commit 1af2b2bfe65e346e332fe040ecdd9df54a81eeb5 +Author: Alpar Torok +Date: Thu Apr 4 14:54:21 2019 +0300 + + Mute failing test + + Tracked in #40838 + +commit 9d34164ce649715f2dc8cb4a4978a3590b1b1a80 +Author: Tim Vernum +Date: Thu Apr 4 21:44:03 2019 +1100 + + Add build utility to check cluster health over ssl (#40713) + + By default, in integ tests we wait for the standalone cluster to start + by using the ant Get task to retrieve the cluster health endpoint. + However the ant task has no facilities for customising the trusted + CAs for a https resource, so if the integ test cluster has TLS enabled + on the http interface (using a custom CA) we need a separate utility + for that purpose. + + Backport of: #40573 + +commit d16f86f7abb684d9ee12df4005f5daac69e6c927 +Author: David Roberts +Date: Thu Apr 4 10:43:30 2019 +0100 + + [ML] Add created_by info to usage stats (#40518) + + This change adds information about which UI path + (if any) created ML anomaly detector jobs to the + stats returned by the _xpack/usage endpoint. + + Counts for the following possibilities are expected: + + * ml_module_apache_access + * ml_module_apm_transaction + * ml_module_auditbeat_process_docker + * ml_module_auditbeat_process_hosts + * ml_module_nginx_access + * ml_module_sample + * multi_metric_wizard + * population_wizard + * single_metric_wizard + * unknown + + The "unknown" count is for jobs that do not have a + created_by setting in their custom_settings. + + Closes #38403 + +commit c8047c06444ca043c77ff3b4aa86c531de3d4de8 +Author: Marios Trivyzas +Date: Thu Apr 4 11:31:28 2019 +0200 + + SQL: [Docs] Small fixes for CURRENT_TIMESTAMP docs (#40792) + + - Added square brackets for the optional argument of precision + - Fixed character to lower case after comma + + (cherry picked from commit d2f6f3b9ce36875e2eb6145c50464b4d72f2b1df) + +commit f3c207d27bcc4cec4fc6a8caab835c166fa58bcb +Author: Marios Trivyzas +Date: Wed Apr 3 19:35:37 2019 +0200 + + SQL: Implement CURRENT_TIME/CURTIME functions (#40662) + + After `TIME` SQL data type is introduced, implement + `CURRENT_TIME/CURTIME` functions similarly to CURRENT_TIMESTAMP + that return the system's current time (only, without the date part). + + Closes: #40468 + (cherry picked from commit 9feede781409d0e264ce45951a25b28ff129b187) + +commit 670e76669c30a518aee28a6311a9b9d9066697bf +Author: Adrien Grand +Date: Thu Apr 4 11:40:42 2019 +0200 + + Fix alias resolution runtime complexity. (#40263) (#40788) + + A user reported that the same query that takes ~900ms when querying an index + pattern only takes ~50ms when only querying indices that have matches. The + query is a date range query and we confirmed that the `can_match` phase works + as expected. I was able to reproduce this issue locally with a single node: with + 900 1-shard indices, a query to an index pattern that matches all indices runs + in ~90ms while a query to the only index that has matches runs in 0-1ms. + + This ended up not being related to the `can_match` phase but to the cost of + resolving aliases when querying an index pattern that matches lots of indices. + In that case, we first resolve the index pattern to a list of concrete indices + and then for each concrete index, we check whether it was matched through an + alias, meaning we might have to apply alias filters. Unfortunately this second + per-index operation runs in linear time with the number of matched concrete + indices, which means that alias resolution runs in O(num_indices^2) overall. + So queries get exponentially slower as an index pattern matches more indices. + + I reorganized alias resolution into a one-step operation that runs in linear + time with the number of matches indices, and then a per-index operation that + runs in linear time with the number of aliases of this index. This makes alias + resolution run is O(num_indices * num_aliases_per_index) overall instead. When + testing the scenario described above, the `took` went down from ~90ms to ~10ms. + It is still more than the 0-1ms latency that one gets when only querying the + single index that has data, but still much better than what we had before. + + Closes #40248 + +commit 65cca2ee6fc6c9614977672fbdfb1185efbb846f +Author: Dimitris Athanasiou +Date: Thu Apr 4 12:28:06 2019 +0300 + + [7.x][ML] Scrolling datafeed should clear scroll contexts on error (#40773) (#40794) + + Closes #40772 + +commit 25944c4317bc3499ab5659993df3fc18ae9e01da +Author: Alpar Torok +Date: Thu Apr 4 11:41:38 2019 +0300 + + convert modules to use testclusters (#40804) + + * convert modules to use testclusters + * Eliminate PluginPropertiesTask and move logic in plugin where it belongs + +commit 6ac307d4689445d166df96fc91a3fc7d0a823256 +Author: Alexander Reelsen +Date: Thu Apr 4 10:28:01 2019 +0200 + + Replace javax activation with jakarta activation (#40247) + + The eclipse foundation has taken over the javax.activation dependency, which + resulted in a naming change of the dependency. + +commit 8ca7325f6537898fe6989edf2bcd9860221068b8 +Author: Alan Woodward +Date: Thu Apr 4 09:07:42 2019 +0100 + + Document restrictions on fuzzy matching when using synonyms (#40783) + + Relates to #25518 + +commit f5f5c3e4296bb2226eed86656e4af5520deca064 +Author: Adrien Grand +Date: Thu Apr 4 10:07:55 2019 +0200 + + Add unit test for MetaDataMappingService with typeless put mapping. (#40578) (#40720) + + This is currently only tested via REST tests. + + Closes #37450 + +commit 4296ff2fd1dd66d0540377ea77d56304fc58323f +Author: Alan Woodward +Date: Thu Apr 4 09:03:08 2019 +0100 + + Test that no-index synonyms can be used with the Analyze API (#40781) + + Relates to #23943 + +commit a6faf85f26c0b7fd2f7951860efff7ab176c2560 +Author: Przemyslaw Gomulka +Date: Thu Apr 4 08:41:28 2019 +0200 + + Migrate systemd packaging tests from bats to java backport(#39954) (#40763) + + Migrating systemd bats tests from bats to java dsl. This also covers + partially the sysv, but more must be added + + relates #32143 + backport #39954 + +commit e5cec876971cba2137e4c61261a249bb2ef71bdc +Author: Martijn van Groningen +Date: Thu Apr 4 08:29:42 2019 +0200 + + Remove -Xlint exclusions in all plugins. (#40721) + + The xlint exclusions of the following plugins were removed: + * ingest-attachment. + * mapper-size. + * transport-nio. Removing the -try exclusion required some work, because + the NettyAdaptor implements AutoCloseable and NettyAdaptor#close() method + could throw an InterruptedException (ChannelFuture#await() and a generic + Exception is re-thrown, which maybe an ChannelFuture). The easiest way + around this to me seemed that NettyAdaptor should not implement AutoCloseable, + because it is not directly used in a try-with-resources statement. + + Relates to #40366 + +commit 1a30ab22fb1500f354eb9c69bfe48145b83d70d7 +Author: Tim Vernum +Date: Thu Apr 4 14:40:15 2019 +1100 + + Show SSL usage when security is not disabled (#40761) + + It is possible to have SSL enabled but security disabled if security + was dynamically disabled by the license type (e.g. trial license). + + e.g. In the following configuration: + + xpack.license.self_generated.type: trial + # xpack.security not set, default to disabled on trial + xpack.security.transport.ssl.enabled: true + + The security feature will be reported as + + available: true + enabled: false + + And in this case, SSL will be active even though security is not + enabled. + + This commit causes the X-Pack feature usage to report the state of the + "ssl" features unless security was explicitly disabled in the + settings. + + Backport of: #40672 + +commit 9d785e2b697e6075ffabfd512e7b6b29f894ceb3 +Author: Ryan Ernst +Date: Wed Apr 3 16:55:03 2019 -0700 + + Don't replace forbidden pattern failures when found (#40710) + + This commit fixes a bug in forbidden patterns where the failures for a + file replace the failures from the previous files instead of extending + them. + +commit a28d5f35d9a03e4f0c644f981fa8011590820c93 +Author: Ryan Ernst +Date: Wed Apr 3 16:25:57 2019 -0700 + + Fix geo points missing test (#40704) + + This commit initializes the geo points for the missing doc values test. + + fixes #40684 + +commit a94e9500ac38b0f4604bc09dcb5601425ae67f7f +Author: Mayya Sharipova +Date: Mon Apr 1 06:25:06 2019 -0400 + + Correct bug in ScriptDocValues (#40488) + + If a field `field_name` was missing in a document, + doc['field_name'].get(0) incorrectly retrieved + a value of the previously accessed document. + This happened because `get(int index)` function + was just accessing `values[index]` without + checking the number of values - `count`. + + This PR fixes this. + +commit 347e059fdc592f99761c9d64fb6d72f6b252e0f4 +Author: James Rodewig +Date: Wed Apr 3 14:55:25 2019 -0400 + + [DOCS] Consistently document dynamic monitoring collection settings (#40598) (#40808) + +commit 6ae7d593eac7967bfb155fb16122fb4d32a03809 +Author: Yannick Welsch +Date: Wed Apr 3 20:27:49 2019 +0200 + + Avoid background sync on relocated primary (#40800) + + There were some test failures caused by the background retention lease sync running on a relocated + primary. This commit fixes the situation that triggered the assertion and reactivates the failing test. + + Closes #40731 + +commit 47a3c42bf216baee4ed18e7ccd1a561ec806a478 +Author: Mark Vieira +Date: Tue Apr 2 20:16:54 2019 -0700 + + Upgrade to latest build scan plugin (#40702) + +commit 050c0ec21e56df7a62d7c7f979f466e94c803f0b +Author: James Rodewig +Date: Wed Apr 3 13:49:20 2019 -0400 + + [DOCS] Note index rollover is not automatically monitored (#40464) + +commit 41a2b37a55815d6967f552a57709df6809530229 +Author: Jake Landis +Date: Wed Apr 3 11:33:55 2019 -0500 + + release notes for 7.0.0-rc2 (#40796) + +commit fd6c5db938aa2fb57b23e79e41eb3190ee20d9c1 +Author: michaelbaamonde +Date: Wed Mar 27 09:17:09 2019 -0400 + + Add release notes for 7.0.0-rc1. (#40479) + + * Add release notes for 7.0.0-rc1. + + * [DOCS] Fixes broken link to breaking changes + + * [DOCS] Removed old ML PRs; edited titles + + * Remove superseded PR. + + * Clean up Lucene upgrade PRs/issues. + +commit 95395938c87d5a9878a725c26e8d09e428a7bf23 +Author: Alpar Torok +Date: Wed Apr 3 19:31:15 2019 +0300 + + Testclusters support for multi node clusters (#40699) + + * Testclusters: introduce support for a multi node cluster + +commit 89389197b35bc1a186597e3ee48156b740d69582 +Author: Christoph Büscher +Date: Wed Apr 3 17:50:42 2019 +0200 + + Help Eclipse infering lambda parameter types (#40747) + + The Eclipse compiler (4.10, Photon) cannot build this test because it cannot + correctly infer the type arguments of the functions. Explicitely adding them + helps in this case. + +commit 3844da318fb09afbb395bd410ddc4ed082e17f6d +Author: Marios Trivyzas +Date: Wed Apr 3 16:42:20 2019 +0200 + + SQL: Fix deserialisation issue of TimeProcessor (#40776) + + TimeProcessor didn't implement `getWriteableName()` so the one from + the parent was used which returned the `NAME` of the parent. This + caused `TimeProcessor` objects to be deserialised into + DateTimeProcessor. + + Moreover, added a restriction to run the TIME related integration tests + only in UTC timezone. + + Fixes: #40717 + + (cherry picked from commit cfea348bec20e547df72c415cccd85279accb767) + +commit b9c46d1dfc400e488b1d399e9474f0dcde02b229 +Author: Ryan Ernst +Date: Wed Apr 3 07:21:13 2019 -0700 + + Improve GCS docs for using keystore (#40605) + + This commit tweaks the wording on using the keystore to store GCS + credentials to note it requires a different key type/command. + + closes #39993 + +commit 09ba3ec6777c8ebf1669db7ee6a162be6b644b6f +Author: Christoph Büscher +Date: Wed Apr 3 14:15:40 2019 +0200 + + Small refactorings to analysis components (#40745) + + This change adds the following internal refactorings: + + * wraps input analyzers into an unmodifiable map in IndexAnalyzers ctor + * removes duplicated indexSetting in IndexAnalyzers + * removes references to IndexAnalyzers from DocumentMapperParser and TypeParser.ParserContext. + It can always be retrieve it from MapperService directly in those cases + +commit 324fef0548f188c3090b6f9502405ab7b9443c8f +Author: David Roberts +Date: Mon Apr 1 17:01:47 2019 +0100 + + [TEST] Mute WatchMetadataTests.testWatchMetadata + + Due to https://github.com/elastic/elasticsearch/issues/40631 + +commit 952c4d9653fb9af4d9af640c558b5680981347b9 +Author: Marios Trivyzas +Date: Wed Apr 3 13:28:30 2019 +0200 + + SQL: Fix display size for DATE/DATETIME (#40669) + + A full format for a DATETIME would be: + `2019-03-30T10:20:30.123+10:00` which is 29 chars long. + + For DATE a full format would be: `2019-03-30T00:00:00.000+10:00` + which is also 29 chars long. + + + (cherry picked from commit 6be83964ed025528778bca8d35692762e166983b) + +commit 1d2bc85586603ad74cc36fe9407c286c10f6610a +Author: David Turner +Date: Wed Apr 3 11:07:10 2019 +0100 + + Inline TransportReplAction#registerRequestHandlers (#40762) + + It is important that resync actions are not rejected on the primary even if its + `write` threadpool is overloaded. Today we do this by exposing + `registerRequestHandlers` to subclasses and overriding it in + `TransportResyncReplicationAction`. This isn't ideal because it obscures the + difference between this action and other replication actions, and also might + allow subclasses to try and use some state before they are properly + initialised. This change replaces this override with a constructor parameter to + solve these issues. + + Relates #40706 + +commit 31e79a73d75c243fc7ff45d540bba596702130c0 +Author: Hendrik Muhs +Date: Wed Apr 3 12:49:11 2019 +0200 + + add HLRC protocol tests for transform state and stats (#40766) + + adds HLRC protocol tests for state and stats hrlc clients + +commit df65e46d107ddb4dc7d7e12cd384891bc3373940 +Author: Jason Tedor +Date: Wed Apr 3 06:39:40 2019 -0400 + + Deprecate versions of Java prior to Java 11 (#40756) + + This commit deprecates versions of Java prior to Java 11. This commit + will cause a warning to be printed to standard error when any command + line tool is invoked, or when Elasticsearch is started. Additionally, we + log a deprecation message when Elasticsearch is started. + +commit e64524c46fa3c9d36922eade57507d264a2d051d +Author: David Turner +Date: Wed Apr 3 08:32:57 2019 +0100 + + Remove some abstractions from `TransportReplicationAction` (#40706) + + `TransportReplicationAction` is a rather complex beast, and some of its + concrete implementations do not need all of its features. More specifically, it + (a) chases a primary around the cluster until it manages to pin it down and + then (b) executes an action on that primary and all its replicas. There are + some actions that are coordinated by the primary itself, meaning that there is + no need for the chase-the-primary phases, and in the case of peer recovery + retention leases and primary/replica resync it is important to bypass these + first phases. + + This commit is a step towards separating the `TransportReplicationAction` into + these two parts. It is a mostly mechanical sequence of steps to remove some + abstractions that are no longer in use. + +commit 4c8c4e595182ee940788a3ecb1d121d293650dd5 +Author: Jim Ferenczi +Date: Wed Apr 3 09:41:50 2019 +0200 + + remove experimental label from search_as_you_type documentation (#40744) + +commit 1f947054ff757a47fadf17c341736a83d476016d +Author: Hendrik Muhs +Date: Tue Apr 2 20:40:55 2019 +0200 + + add reason to DataFrameTransformState and add hlrc protocol tests (#40736) + + add field "reason" to DataFrameTransformState, add hlrc protocol tests and allow unknown fields for DataFrameTransformState + +commit f377155f10aac1fa75254789243b57f44d8e5818 +Author: Jason Tedor +Date: Tue Apr 2 17:55:21 2019 -0400 + + Use default memory lock setting in testing (#40730) + + Today we are running our internal tests with bootstrap.memory_lock + enabled. This is not out default setting, and not the recommended + value. This commit switches to use the default value, which is to not + enable bootstrap.memory_lock. + +commit fd24f1de2894576fed40815e57c468280edc147f +Author: Nik Everett +Date: Tue Apr 2 16:09:48 2019 -0400 + + Docs: Move id in the java-api (#40748) + + Moves the id of the preface in the java-api so it is compatible with + both AsciiDoc and Asciidoctor. As it stands now we apply the id that we + want for the preface to the book itself which is strange and only works + with AsciiDoc. + +commit 945e7ca01e6bd418b3e7f4d3c09450325629cf78 +Author: Benjamin Trent +Date: Tue Apr 2 14:16:55 2019 -0500 + + [ML] Periodically persist data-frame running statistics to internal index (#40650) (#40729) + + * [ML] Add mappings, serialization, and hooks to persist stats + + * Adding tests for transforms without tasks having stats persisted + + * intermittent commit + + * Adjusting usage stats to account for stored stats docs + + * Adding tests for id expander + + * Addressing PR comments + + * removing unused import + + * adding shard failures to the task response + +commit abbfc75052ced4af3196b991c6f8cfc387383aae +Author: Zachary Tong +Date: Tue Apr 2 14:24:41 2019 -0400 + + Remove timezone validation on rollup range queries (#40647) + + We enforced the timezone of range queries when using the rollup + search endpoint, but this validation is not needed. Since + rollup dates are stored in UTC, and range queries are always + converted to UTC (even if specifying a `time_zone`) the validation + is not needed and can prevent legitimate queries from running. + +commit 4842d7fb7d6b8b6e9bc5ef7848219f8e3cb7485f +Author: Benjamin Trent +Date: Tue Apr 2 12:33:51 2019 -0500 + + [ML] addressing test failure (#40701) (#40728) + + * [ML] Fixing test + + * adjusting line lengths + + * marking valid seqno as final + +commit 29180cefacb90406ef9206408f24d8b354c4722e +Author: Benjamin Trent +Date: Tue Apr 2 12:33:35 2019 -0500 + + [ML] fix test check as randomness allows for different hours (#40536) (#40727) + + * [ML] fix test check as randomness allows for different hours + + * Re-enabling test + +commit 2fd01cc0b76f637705bacad8ce7a1897c15412c2 +Author: Lee Hinman +Date: Tue Apr 2 10:56:24 2019 -0600 + + Fix testRunStateChangePolicyWithAsyncActionNextStep race condition (#40707) + + Previously we only set the latch countdown with `nextStep.setLatch` after the + cluster state change has already been counted down. However, it's possible + execution could have already started, causing the latch to be missed when the + `MockAsyncActionStep` is being executed. + + This moves the latch setting to be before the call to + `runPolicyAfterStateChange`, which means it is always available when the + `MockAsyncActionStep` is executed. + + I was able to reproduce the failure every 30-40 runs before this change. With + this change, running 2000+ times the test passes. + + Resolves #40018 + +commit ee8382d9cb8a876b77e7c5822622dd219ccdaeaa +Author: Omkar Deshpande +Date: Sat Mar 30 02:49:26 2019 +0530 + + Update version numbers in Elasticsearch-Py docs (#40355) + +commit dd624c31b0bca498707d75fdf93bd8e44707de49 +Author: Simon Willnauer +Date: Tue Apr 2 16:13:53 2019 +0200 + + Don't mark shard as refreshPending on stats fetching (#40458) + + Completion and DocStats are pulled from internal readers + instead of external since #33835 and #33847 which doesn't require + us to refresh after a stats call since refreshes will happen internally + anyhow and that will cause updated stats on ongoing indexing. + +commit b78d63d6244c79d2bf6d5f7e921008a1f77d4377 +Author: Marios Trivyzas +Date: Tue Apr 2 15:21:38 2019 +0200 + + SQL: [TEST] Mute TIME related failing tests + + Mute failing tests tracked by: #40717 + + (cherry picked from commit 23719bbd687546500f1b2d2b86069a7efbf5bd29) + +commit a3f53cfd05666dca4ca06730ed332bb1e1bdca18 +Author: Dimitrios Liappis +Date: Tue Apr 2 16:23:22 2019 +0300 + + [TEST] Mute RecoveryWithConcurrentIndexing test (#40735) + + Due to https://github.com/elastic/elasticsearch/issues/40731 + + Backport of https://github.com/elastic/elasticsearch/pull/40733 + +commit 91a74ad7991623b6bfbcd1d6600f86d8dab55d8c +Author: Jason Tedor +Date: Tue Apr 2 08:53:45 2019 -0400 + + Add migration note for Ubuntu 14.04 + + This commit adds a migration note to the docs for Ubuntu 14.04 seeing as + it will not be supported after April 30, 2019 by upstream. + +commit 60000400db2ab9752fc84460f822888234db28b3 +Author: Jason Tedor +Date: Tue Apr 2 08:51:11 2019 -0400 + + Drop support for Ubuntu 14.04 (#40709) + + With the 7.0.0 release, we are dropping support for Ubuntu 14.04. This + commit removes this OS from our testing infrastructure. + +commit 6f00952abda52cd6f5f0a02a461cbd52c1072fbe +Author: David Turner +Date: Tue Apr 2 13:18:17 2019 +0100 + + Use TAR instead of DOCKER build type before 6.7.0 (#40723) + + In 6.7.0 (#39378) we added a build type of DOCKER for the docker images, but + unfortunately earlier versions do not understand this and will reject any + transport messages that mention this build type. + + This commit fixes this by reporting TAR instead of DOCKER when talking to older + nodes. + + Relates (but does not fix) #40511 + Relates #39378 + +commit 4772b5d636878342c0661672ef32ee452e993417 +Author: Benjamin Trent +Date: Tue Apr 2 07:04:33 2019 -0500 + + [ML] Addressing bug streaming DatafeedConfig aggs from (<= 6.5.4) -> 6.7.0 (#40659) + + * [ML] Addressing bug streaming DatafeedConfig aggs from (<= 6.5.4) -> 6.7.0 (#40610) + + * Addressing stream failure and adding tests to catch such in the future + + * Add aggs to full cluster restart tests + + * Test BWC for datafeeds with and without aggs + + The wire serialisation is different for null/non-null + aggs, so it's worth testing both cases. + + * Fixing bwc test, removing types + + * Fixing BWC test for datafeed + + * Update 40_ml_datafeed_crud.yml + + * Update build.gradle + +commit 2c770ba3cb66b345e5029ae146c7c5336d902fee +Author: Tim Vernum +Date: Tue Apr 2 20:55:10 2019 +1100 + + Support mustache templates in role mappings (#40571) + + This adds a new `role_templates` field to role mappings that is an + alternative to the existing roles field. + + These templates are evaluated at runtime to determine which roles should be + granted to a user. + For example, it is possible to specify: + + "role_templates": [ + { "template":{ "source": "_user_{{username}}" } } + ] + + which would mean that every user is assigned to their own role based on + their username. + + You may not specify both roles and role_templates in the same role + mapping. + + This commit adds support for templates to the role mapping API, the role + mapping engine, the Java high level rest client, and Elasticsearch + documentation. + + Due to the lack of caching in our role mapping store, it is currently + inefficient to use a large number of templated role mappings. This will be + addressed in a future change. + + Backport of: #39984, #40504 + +commit 965e3110945212ac0c9fb55fceeadc861061b605 +Author: Adrien Grand +Date: Tue Apr 2 11:18:53 2019 +0200 + + Update indexing speed recommendations around the refresh interval. (#40690) + + We now need to update recommendations now that we have introduced the concept + of "search idle" shards. + +commit c644fbfc6ec71926423760d801ec800d8a2abfc5 +Author: Alexander Reelsen +Date: Tue Apr 2 10:26:54 2019 +0200 + + Allow single digit milliseconds in strict date parsing (#40676) + + In order to remain compatible with the existing joda based + implementation the parsing of milliseconds should support parsing single + digits instead of relying on three, even with strict formats. + + This adds a few tests to duel against the existing joda based + implementation in order to ensure the parsing behaviour is the same. + + Closes #40403 + +commit 7bdd41399d76a64b63fd360136184a7aef8fa0fd +Author: Tim Vernum +Date: Tue Apr 2 14:48:39 2019 +1100 + + Support roles with application privileges against wildcard applications (#40675) + + This commit introduces 2 changes to application privileges: + + - The validation rules now accept a wildcard in the "suffix" of an application name. + Wildcards were always accepted in the application name, but the "valid filename" check + for the suffix incorrectly prevented the use of wildcards there. + + - A role may now be defined against a wildcard application (e.g. kibana-*) and this will + be correctly treated as granting the named privileges against all named applications. + This does not allow wildcard application names in the body of a "has-privileges" check, but the + "has-privileges" check can test concrete application names against roles with wildcards. + + Backport of: #40398 + +commit dfc70e6ef0746242464e8bbec10ebbeead89b914 +Author: Christoph Büscher +Date: Tue Apr 2 01:43:02 2019 +0200 + + Correct indention in synonym docs (#40711) + + The stopword filter should be on the same level as the synonym filter in the + example request. Correcting this for better readability. + +commit 899ed2bf816b62d8f8e141b7325bb0f6ce9f5e93 +Author: Marios Trivyzas +Date: Mon Apr 1 23:30:39 2019 +0200 + + SQL: Introduce SQL TIME data type (#39802) + + Support ANSI SQL's TIME type by introductin a runtime-only + ES SQL time type. + + Closes: #38174 + (cherry picked from commit 046ccd4cf0a251b2a3ddff6b072ab539a6711900) + +commit 12bf3b2025bc494b94b14ee65f35e98f0b981962 +Author: Jay Modi +Date: Mon Apr 1 14:23:24 2019 -0600 + + Remove with(out)-system-key tests (#40547) (#40603) + + This change removes the variants of the rolling upgrade and full + cluster restart tests that use or do not use a system key. These tests + were added during 5.x when the system key was still used for security + and now the system key is only used as the watcher encryption key so + duplicating rolling upgrade and full cluster restarts is not needed. + + The change here removes the subprojects for testing these scenarios and + defaults to always run with the watcher sensitive values encrypted for + these tests. + +commit e88b86219285f9f78384ef171942db4a73333134 +Author: Andy Bristol +Date: Mon Apr 1 12:03:25 2019 -0700 + + fix summary of phrase_prefix scoring (#40567) + + The language here implies that phrase_prefix scoring works like + most_fields, but it actually works like best_fields + +commit 7aa3cf54454d7d53e2c945b47d3665f3a38cfc07 +Author: lcawl +Date: Mon Apr 1 12:01:13 2019 -0700 + + [DOCS] Adds placeholder for 7.0.0-rc2 release notes + +commit 575918e8e6921bcfe8ae67020ff05f0ad4705605 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Apr 1 19:57:17 2019 +0200 + + Reindex from Remote allow date math (#40303) + + Previously, reindexing from remote using date math in the source index + name did not work if the math contained / or ,. A workaround was to + then URL escape the index name in the request. + + With this change, we now support any index name in the remote request + that the remote source supports, doing the URL escape when sending the + request. + + Related to #23533 + +commit edb0d42b41f21838a0d142dac4f64b91e4055b88 +Author: lcawl +Date: Mon Apr 1 10:34:51 2019 -0700 + + [DOCS] Add notable release highlights for 7.0 + +commit 919358edecb98b42c1bd55e41fca115c7eee03ab +Author: Jack Conradson +Date: Mon Apr 1 10:17:16 2019 -0700 + + Update casting table for Painless docs (#40491) + + This updates the casting table to reflect the recent changes for casting consistency in Painless. This also adds a small section on explicitly casting a character to a String which has always been allowed but undocumented. + +commit 16a92689fc89a1ad3753a731b44ac3df81b4b314 +Author: Nik Everett +Date: Mon Apr 1 13:02:23 2019 -0400 + + Docs: Drop inline callouts from the java API (#39932) (#40693) + + There is a single example in the Java API docs that contains an inline + callout that is incompatible with Asciidoctor: + + ``` + client.prepareUpdate("ttl", "doc", "1") + .setScript(new Script( + "ctx._source.gender = \"male\"" <1> , ScriptService.ScriptType.INLINE, null, null)) + .get(); + ``` + + This rewrites the example to use an Asciidoctor compatible end of line + callout. It also looks nicer to me because it fits better on the page. + + ``` + client.prepareUpdate("ttl", "doc", "1") + .setScript(new Script( + "ctx._source.gender = \"male\"", <1> + ScriptService.ScriptType.INLINE, null, null)) + .get(); + ``` + +commit dfe4c6c568975c9145f70c61ab312d0b2981f5df +Author: Andrei Stefan +Date: Mon Apr 1 18:10:51 2019 +0300 + + SQL: have LIKE/RLIKE use wildcard and regexp queries (#40628) + + * Have LIKE and RLIKE only use term-level queries (wildcard and regexp respectively). They + are already working only with exact fields, thus be in-line with how + SQL works in general (what you index is what you search on). + + (cherry picked from commit 1bba887d481b49db231a1442922f1813952dcc67) + +commit f8d3d685e5a17582461410148f4dcf6d7eca4aa4 +Author: Andrei Stefan +Date: Mon Apr 1 18:01:03 2019 +0300 + + SQL: Documentation for LIKE and RLIKE operators (#40623) + + (cherry picked from commit 9536c5f7b7fecda4592f6b169adef180eadbba54) + +commit 8cd96ee16081cdb4c573bd0250ada956075be3c4 +Author: Andrei Stefan +Date: Mon Apr 1 17:34:54 2019 +0300 + + Add JDBC trustore type to the docs (#40592) + + (cherry picked from commit cafde31129d85d20f9e445be131c9e85163bed9d) + +commit 655e3d8f7567fa3823cd0cc5c0da28559bcdbcd1 +Author: Benjamin Trent +Date: Mon Apr 1 10:00:32 2019 -0500 + + [ML] fix test, should account for async nature of audit (#40637) (#40683) + +commit 4b3b0029422a2883ddb1077fdb8745a2b76637e1 +Author: Martijn van Groningen +Date: Mon Apr 1 16:20:20 2019 +0200 + + Remove -Xlint exclusions in the ingest-geoip module. (#40629) + + Relates to #40366 + +commit 54b0ff8e337bdf64f0ca1b348b15dfbf2f3847af +Author: David Roberts +Date: Mon Apr 1 14:20:01 2019 +0100 + + [TEST] Mute WebhookHttpsIntegrationTests.testHttps + + Due to https://github.com/elastic/elasticsearch/issues/35503 + +commit 2b4ebcab53e1aaf873bea0b2c9d4c7eeeba5fec1 +Author: James Rodewig +Date: Mon Apr 1 08:47:12 2019 -0400 + + [DOCS] Add 'time value' links to several monitor settings (#40633) (#40685) + +commit 39756fa538b989d2af12a9db2068e36a4189a12c +Author: James Rodewig +Date: Mon Apr 1 08:25:26 2019 -0400 + + [DOCS] Correct keystore commands for Email and Jira actions in Watcher (#40417) (#40612) + +commit 287e334ef30d22449605e1e94d5ea5d4e6be30a2 +Author: Andrey Ershov +Date: Mon Apr 1 11:49:05 2019 +0300 + + Do not perform cleanup if Manifest write fails with dirty exception (#40519) + + Currently, if Manifest write is unsuccessful (i.e. WriteStateException + is thrown) we perform cleanup of newly created metadata files. + However, this is wrong. + Consider the following sequence (caught by CI here + https://github.com/elastic/elasticsearch/issues/39077): + + - cluster global data is written **successful** + - the associated manifest write **fails** (during the fsync, ie files + have been written) + - deleting (revert) the manifest files, **fails**, metadata is + therefore persisted + - deleting (revert) the cluster global data is **successful** + + In this case, when trying to load metadata (after node restart + because of dirty WriteStateException), the following exception will + happen + ``` + java.io.IOException: failed to find global metadata [generation: 0] + ``` + because the manifest file is referencing missing global metadata file. + + This commit checks if thrown WriteStateException is dirty and if its + we don't perform any cleanup, because new Manifest file might be + created, but its deletion has failed. + In the future, we might add more fine-grained check - perform the + clean up if WriteStateException is dirty, but Manifest deletion is + successful. + + Closes https://github.com/elastic/elasticsearch/issues/39077 + + (cherry picked from commit 1fac56916bb3c4f3333c639e59188dbe743e385b) + +commit 7cc79123df65b763b437cc899cf2af3778b32df1 +Author: Jim Ferenczi +Date: Mon Apr 1 08:41:08 2019 +0200 + + Fix merging of text field mapper (#40627) + + On mapping updates the `text` field mapper does not update + the field types for the underlying prefix and phrase fields. + In practice this shouldn't be considered as a bug but we have + an assert in the code that check that field types in the mapper service + are identical to the ones present in field mappers. + +commit 74bf4b829220d6bd5ae587591365549e677d4841 +Author: Ricky(道离) +Date: Mon Apr 1 14:00:28 2019 +0800 + + If not set JAVA_HOME then use the JDK that Gradle was run with instead (for master) (#40622) + + Extending the IDE behavior to the cli + +commit e596b33ce8a3bf52bec34594f3e547449c9e0f7b +Author: Jason Tedor +Date: Sun Mar 31 22:10:45 2019 -0400 + + Remove mention of soft deletes from getting started (#40668) + + We no longer need to mention soft deletes in the getting started guide + now that retention leases exist and default to 12h. This commit removes + mention of soft deletes from the getting started guide, to simplify that + content. + +commit cebe5094606b12b925fcc3663659701c6c3c6575 +Author: Jason Tedor +Date: Sun Mar 31 19:42:22 2019 -0400 + + Fix bug in detecting use of bundled JDK on macOS + + This commit fixes a bug in detecting the use of the bundled JDK on + macOS. This bug arose because the path of Java home is different on + macOS. + +commit 567e8f8b63de606095f81d9091b718b07f476370 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Sat Mar 30 00:43:15 2019 +0100 + + Reindex conflicts clarification (docs) (#40442) + + Made it more clear that conflicts : proceed only affects version + conflicts. + +commit 1b95ddfa3b389809e3ffc99edac87124024803f9 +Author: Marios Trivyzas +Date: Sat Mar 30 16:56:44 2019 +0100 + + SQL: [Tests] Enable integration tests for fixed issues (#40664) + + Enable some Ignored integration tests for issues/features that + have already been resolved/implemented. + + (cherry picked from commit c23580f477ffc61c5701e14a91006db7bf21a8d4) + +commit 1e90b29c00dcde34139cb28d7f520e8c1f2793bb +Author: Mayya Sharipova +Date: Sat Mar 30 10:56:23 2019 -0400 + + Add information about the default sort mode (#40657) + +commit 862fe612e0ceef0c7c6a1979a4df55c34d4b698f +Author: Marios Trivyzas +Date: Sat Mar 30 12:51:13 2019 +0100 + + SQL: Fix precedence of `::` psql like CAST operator (#40665) + + Previously, an expression like `10 + 2::long` would be interpreted + as `CAST(10 + 2 AS LONG)` instead of `10 + CAST(2 AS LONG)`. + + + (cherry picked from commit e34cc2f38b1477e78788ee377938f42cc47187c7) + +commit 31776a4b155f29b5dca555f2bbffa159c3d55c16 +Author: Marios Trivyzas +Date: Sat Mar 30 12:41:58 2019 +0100 + + SQL: [Docs] Fix example for CURDATE + + Follows: 3dd0384d683940871f5d061f153b70c0420150fa + (cherry picked from commit 35d731754455ec91e20190a874592a740c66aa44) + +commit 6edb80271642373cca0a058f2e0af7bb21be7358 +Author: Marios Trivyzas +Date: Sat Mar 30 12:05:15 2019 +0100 + + SQL: [Docs] Fix doc errors regarding CURRENT_DATE. (#40649) + + Some parts wrongly refered to CURRENT_TIMESTAMP. + + (cherry picked from commit 3dd0384d683940871f5d061f153b70c0420150fa) + +commit 4f5a83dba257dddb8b705ea8048f180cfc5b822a +Author: Christoph Büscher +Date: Fri Mar 29 23:38:37 2019 +0100 + + Clarify using time_zone and date math in range query (#40655) + + Currently, the docs correctly state that using `now` in range queries will not + be affected by the `time_zone` parameter. However, using date math roundings + like e.g. `now\d` will be affected by the `time_zone`. Adding this example + because it seems to be a frequently asked question and source of confusion. + + Relates to #40581 + +commit 803ff3bc4c0d0cc4212ab08db1d4ec81385bf6a3 +Author: Ryan Ernst +Date: Fri Mar 29 14:20:53 2019 -0700 + + Add notice for bundled jdk (#40576) + + * Add notice for bundled jdk + + This commit adds the license/notice for the bundled openjdk. + + * First draft + + * iteration + + * Fix package notices + + * Iteration + + * One more iteration + +commit a0e3af80fffc509b35f586231584d19a748b8f33 +Author: Ryan Ernst +Date: Fri Mar 29 13:24:08 2019 -0700 + + disable kerberos test until kerberos fixture is working again + +commit cbe7d335ffd6f88c1d9d3493baeb99c6fbee66de +Author: David Roberts +Date: Fri Mar 29 16:40:25 2019 +0000 + + [DOCS] Use "source" instead of "inline" in ML docs (#40635) + + Specifying an inline script in an "inline" field + was deprecated in 5.x. The new field name is + "source". + + (Since 6.x still accepts "inline" I will only backport + this docs change as far as 7.0.) + +commit 14ee3d3f9525380decdc9bdd80330c9f986a9075 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Mar 29 17:30:08 2019 +0100 + + Unmute and fix testSubParserArray (#40626) + + testSubParserArray failed, fixed and improved to not always have an + object as outer-level inside array. + + Closes #40617 + +commit 92d07e937700d9832619ac1b553779b79568561b +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Mar 28 18:55:31 2019 +0100 + + Geo Point parse error fix (#40447) + + When geo point parsing threw a parse exception, it did not consume + remaining tokens from the parser. This in turn meant that + indexing documents with malformed geo points into mappings with + ignore_malformed=true would fail in some cases, since DocumentParser + expects geo_point parsing to end on the END_OBJECT token. + + Related to #17617 + +commit a0b02ce6ef98f170269c727542eee2013c7d048b +Author: Luca Cavanna +Date: Thu Mar 28 13:04:50 2019 +0100 + + Move top-level pipeline aggs out of QuerySearchResult (#40319) + + As part of #40177 we have added top-level pipeline aggs to + `InternalAggregations`. Given that `QuerySearchResult` holds an + `InternalAggregations` instance, there is no need to keep on setting + top-level pipeline aggs separately. Top-level pipeline aggs can then + always be transported through `InternalAggregations`. Such change is + made in a backwards compatible manner. + +commit f6eefd4af78bdb8454f5103ee7d19979409642d7 +Author: Luca Cavanna +Date: Wed Mar 27 17:30:05 2019 +0100 + + Increase suite timeout to 30 minutes for docs tests (#40521) + + I have been hitting the suite timeout on `DocsClientYamlTestSuiteIT` + As far as I can see, the docs tests are taking quite a while, I assume + it's because more and more docs snippets get added over time, which + means more tests. The current suite timeout is the default 20 minutes. + It takes me just a little less than 20 minutes to run these on my + laptop. On my CI, I end up hitting the suite timeout. Hereby I propose + that we increase the suite timeout to 30 minutes. + +commit 444b4c41364182d608226359b9780430c6de3ab8 +Author: Oghenovo Usiwoma <37949128+Eunovo@users.noreply.github.com> +Date: Wed Mar 27 08:03:09 2019 -0700 + + Improve error message for absence of indices (#39789) + + "no indices exist" has been added to the error message for absence of indices + +commit f2ca45c2107c5eb6ee905900bcdfe29c6569dd4b +Author: Luca Cavanna +Date: Mon Mar 25 10:26:55 2019 +0100 + + Add integration tests to verify CCS output (#40038) + + We recently introduced the option to minimize network roundtrips when + executing cross-cluster search requests. All the changes made around + that are separately unit tested, and there are some yaml tests that + exercise the new code-path which involves multiple coordination steps. + This commit adds new integration tests that compare the output given by + CCS when running the same queries using the two different execution + modes available. + + Relates to #32125 + +commit f7e3f748b6ed67070f8b8fc87c8cb194bb7287ed +Author: aladdam +Date: Wed Mar 20 08:19:32 2019 -0700 + + [DOCS] match description with example in allocation docs page (#39606) + +commit 48b0deef4fc95b688ea9f683edd3e9c75374c272 +Author: Luca Cavanna +Date: Wed Mar 20 16:14:08 2019 +0100 + + Remove throws IOException from PipelineAggregationBuilder#create (#40222) + + IOException are never thrown in any of the existing pipeline aggregation + builders. Removing the throws IOException from the create method allows + to remove it also from a couple of other methods which ends up simplifying + AggregationPhase (one less catch). + +commit 336c92e4d3510632e46f3d9750a422aa0e1a18c9 +Author: krisds +Date: Wed Mar 20 16:16:23 2019 +0100 + + [Contradictory type checks](https://lgtm.com/projects/g/elastic/elasticsearch/snapshot/dist-1916470085-1548143539391/files/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/debug/Debug.java#xe223478b74721d35:1) (#38154) + +commit 1edebfc8345147cb5c56a3592a01ce992ffcdd84 +Author: avshiav <30467621+avshiav@users.noreply.github.com> +Date: Wed Mar 20 17:18:01 2019 +0200 + + [DOCS] Fixed missing space (#38305) + +commit 05ed1cfb33111595402e7002ae0619552a6bf311 +Author: krisds +Date: Thu Mar 21 10:48:31 2019 +0100 + + Remove unused variable in QueryContainer + +commit 293297ae3da13d4c121cc1c0c8676fada3aa5bac +Author: Alpar Torok +Date: Fri Mar 29 16:46:55 2019 +0200 + + Fix repository-hdfs when no docker and unnecesary fixture + + The hdfs-fixture is actually executed in plugin/repository-hdfs as a + dependency. The fixture is not needed and actually causes a failure + because we have two copies now and both use the same ports. + +commit 2b91fb1cc0391dbf1d41674bfb497b8b817ebcf3 +Author: Alpar Torok +Date: Fri Mar 29 14:13:18 2019 +0200 + + Avoid building hdfs-fixure use an image that works instead + + Avoid the additional requirement for the debian package repos to be up, + and depend on dockerhub only instead. + +commit cb5e48a842e79c2f46eaae1e006cfaf498382187 +Author: Jason Tedor +Date: Fri Mar 29 10:44:08 2019 -0400 + + Fix more broken links in plugins docs + + The archive docs were moved around a bit, and this commit fixes another + broken link in the plugins docs. + +commit 443669c64b83dcf682f43eef995f97b1bedf0a7c +Author: Jason Tedor +Date: Fri Mar 29 10:18:59 2019 -0400 + + Fix archives links in plugins docs + + This commit fixes some broken links to the archive layouts in the + plugins docs. + +commit 7e8f1c2da735edaba927df60be720eeb506d01fe +Author: Ryan Ernst +Date: Fri Mar 29 06:04:46 2019 -0700 + + Add docs for bundled jdk (#40487) + + This commit changes the note in docs about required java version to note + the existence of the bundled jdk and how to bring your own java. It also + reorganizes the zip/targz docs as zip is no longer suitable on + Linux/MacOS. + +commit 585f38787ceb8833bdeb4f976c96c92373e924f1 +Author: Jason Tedor +Date: Fri Mar 29 08:24:52 2019 -0400 + + Add usage indicators for the bundled JDK (#40616) + + This commit adds indications whether or not a distribution is from the + bundled JDK, and whether or not we are using the bundled JDK. + +commit e8c0b537963242addbad934dcb9ed532963a8e16 +Author: Alpar Torok +Date: Fri Mar 29 12:07:02 2019 +0200 + + Add ability to mute and mute flaky fixture (#40630) + +commit ae65f77adda8459aca3cd6939585813753b8adcb +Author: Jim Ferenczi +Date: Fri Mar 29 09:34:51 2019 +0100 + + Update docs for the DFR similarity (#40579) + + The basic models `b, de, p` and the after effect `no` + are not available anymore in Lucene 8 but they are still + listed in the >7x documentation. This change removes these + references that should also be listed in the breaking change + of es 7.0. + + Closes #40264 + +commit be318001545aae1ae80ec3b8befccf942cefa8ed +Author: Martijn van Groningen +Date: Fri Mar 29 09:44:42 2019 +0100 + + Update ingest jdocs that a null return value will drop the current document. (#40359) + +commit 89837eb9188ce34aa0e96601e5de97b5b0acb9b7 +Author: Martijn van Groningen +Date: Fri Mar 29 09:42:59 2019 +0100 + + Remove -Xlint exclusions in the ingest-common module. (#40505) + + Fix the generics in processors extending AbstractStringProcessor and its factory. + + Relates to #40366 + +commit e256eb361a314e6e1bf8acffb73976baf8db4947 +Author: Jim Ferenczi +Date: Fri Mar 29 09:01:36 2019 +0100 + + Fix merging of search_as_you_type field mapper (#40593) + + The merge of the `search_as_you_type` field mapper uses the wrong prefix field + and does not update the underlying field types. + +commit 8aca08bf52774148e93d99946f19c49da20f99fe +Author: Ioannis Kakavas +Date: Thu Mar 28 17:35:33 2019 +0200 + + Disable integTest when Docker is not available (#40585) + + * Disable integTest when Docker is not available + + Resolves: #40549 + +commit 49f18811c3b810ea71a4ba388616768bb5581956 +Author: Alpar Torok +Date: Fri Mar 29 08:08:04 2019 +0200 + + Run the build integ test in parallel (#39788) + + * Run the build integ test in parallel + + Because the randomized runner lives in buildSrc, we run these tests with + the Gradle runner, and had no parallelism configured so far. + + * Handle Windows and "auto" better + +commit 35d96c22c0fd209be5db3216f02d675d4f64961a +Author: Alpar Torok +Date: Fri Mar 29 08:01:06 2019 +0200 + + Fix 3rd pary S3 tests (#40588) + + * Fix 3rd pary S3 tests + + This is allready excluded on line 186, by doing this again here, the + other exclusion from arround that line are removed causing the tests to + fail. + + * Fix blacklisting with the fixture + +commit c012c59115916e7f87794e144ce6bfb2f555c7be +Author: Tim Vernum +Date: Fri Mar 29 10:11:00 2019 +1100 + + Improve certutil --pass documentation about empty password (#40574) + + Improve the documentation of parameter --pass of elasticsearch-certutil + + Backport of: #40137 + + Co-Authored-By: Diego Cardozo Sandrim + Co-Authored-By: Vigneash Sundar + +commit a13be65b01681ef53195ee15988312811c5cce4b +Author: Christoph Büscher +Date: Thu Mar 28 22:11:46 2019 +0100 + + Fixing typo in test error message (#40611) + +commit 6c13ed7db866a36cc57491630a59489b90505455 +Author: Jeff Hajewski +Date: Thu Mar 28 16:07:03 2019 -0500 + + Update max dims for vectors to 1024. (#40597) + +commit 7255562afddaada8bf2e0740efd98e905a6ca1b6 +Author: Jason Tedor +Date: Thu Mar 28 16:22:59 2019 -0400 + + Add start and stop time to cat recovery API (#40378) + + The cat recovery API is incredibly useful. Yet it is missing the start + and stop time as an option from the output. This commit adds these as + options to the cat recovery API. We elect to make these not visible by + default to avoid breaking the output that users might rely on. + +commit 6f26156abfe8546820dfbbe105a2db9c7f674b44 +Author: James Rodewig +Date: Thu Mar 28 15:03:14 2019 -0400 + + [DOCS] Document common settings for snapshot repository plugins (#40475) (#40606) + +commit 24755209b4b0f24dfdc7c478acb1063c2cc7dc3a +Author: Mayya Sharipova +Date: Thu Mar 28 11:29:29 2019 -0400 + + Add randomScore function in script_score query (#40186) + + To make script_score query to have the same features + as function_score query, we need to add randomScore + function. + + This function produces different + random scores on different index shards. + It is also able to produce random scores + based on the internal Lucene Document Ids. + +commit 85848af8cf94170104b9396970165027b5b0e6a9 +Author: lcawl +Date: Thu Mar 28 10:03:40 2019 -0700 + + [DOCS] Fixes formatting in breaking changes + +commit db7f00098e441631b7b7314d886d90e7ece213c6 +Author: Gordon Brown +Date: Thu Mar 28 10:53:44 2019 -0600 + + Correct ILM metadata minimum compatibility version (#40569) + + The ILM metadata minimum compatibility version was not set correctly, + which can cause issues in mixed-version clusters. + +commit d1357147f4c006d43b876f35b4d981215c13a6d5 +Author: Lee Hinman +Date: Thu Mar 28 10:42:28 2019 -0600 + + Handle null retention leases in WaitForNoFollowersStep (#40477) + + In some cases the retention leases can return null, causing a + `NullPointerException` when waiting for no followers. + + This wraps those so that no NPE is thrown. + + Here is an example failure: + + ``` + [2019-03-26T09:24:01,368][ERROR][o.e.x.i.IndexLifecycleRunner] [node-0] policy [deletePolicy] for index [ilm-00001] failed on step [{"phase":"delete","action":"delete","name":"wait-for-shard-history-leases"}]. Moving to ERROR step + java.lang.NullPointerException: null + at org.elasticsearch.xpack.core.indexlifecycle.WaitForNoFollowersStep.lambda$evaluateCondition$0(WaitForNoFollowersStep.java:60) ~[?:?] + at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:267) ~[?:1.8.0_191] + at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_191] + at java.util.Spliterators$ArraySpliterator.tryAdvance(Spliterators.java:958) ~[?:1.8.0_191] + at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126) ~[?:1.8.0_191] + at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498) ~[?:1.8.0_191] + at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485) ~[?:1.8.0_191] + at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_191] + at java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230) ~[?:1.8.0_191] + at java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196) ~[?:1.8.0_191] + at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_191] + at java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:449) ~[?:1.8.0_191] + at org.elasticsearch.xpack.core.indexlifecycle.WaitForNoFollowersStep.lambda$evaluateCondition$2(WaitForNoFollowersStep.java:61) ~[?:?] + at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:68) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:64) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onCompletion(TransportBroadcastByNodeAction.java:383) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onNodeResponse(TransportBroadcastByNodeAction.java:352) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:324) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:314) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1095) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1176) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + ... + ``` + +commit 873c5638e6cc0a493a3b9707cd6ca7a668c06b76 +Author: Luca Cavanna +Date: Thu Mar 28 16:53:20 2019 +0100 + + Mute DataFrameAuditorIT#testAuditorWritesAudits + + Relates to #40594 + +commit d791e0893201c8977dd073aa9b530e4436addc96 +Author: Alpar Torok +Date: Thu Mar 28 17:23:40 2019 +0200 + + Test fixtures krb5 (#40297) + + Replaces the vagrant based kerberos fixtures with docker based test fixtures plugin. + The configuration is now entirely static on the docker side and no longer driven by Gradle, + also two different services are being configured since there are two different consumers of the fixture that can run in parallel and require different configurations. + +commit 482d9804b1793929eef0f73aefee55e491dcbd48 +Author: Igor Motov +Date: Thu Mar 28 09:21:59 2019 -0400 + + SQL: Centralize SQL test dependencies version handling (#40551) + + Moves definitions of versions for csvjdbc and h2 libraries to the + central build.gradle file. + +commit fea91c61133844b5da0280817ac8b80c5a3b8c86 +Author: Yannick Welsch +Date: Thu Mar 28 14:03:29 2019 +0100 + + Mute testTracerLog + + Relates to #40586 + +commit 8b9a0067405bb701f2b9dcbc9cd2b3b52e020eed +Author: Yannick Welsch +Date: Thu Mar 28 14:01:11 2019 +0100 + + Mute testHttpInput + + Relates #40587 + +commit fe32ce6009723c9a5eaf91ea76aba0ede7e09d8f +Author: Andrei Stefan +Date: Thu Mar 28 14:28:07 2019 +0200 + + Include functions' aliases in the list of functions (#40584) + + (cherry picked from commit 4034f8f31616353d2f5d09a90753d75e2870ee18) + +commit 1a3916a8de06b84a0d40800818c0984695242857 +Author: David Turner +Date: Thu Mar 28 12:13:25 2019 +0000 + + Optimise rejection of out-of-range `long` values (#40325) + + Today if you try and insert a very large number like `1e9999999` into a long + field we first construct this number as a `BigDecimal`, convert this to a + `BigInteger` and then reject it because it is out of range. Unfortunately + making such a large `BigInteger` is rather expensive. + + We can avoid this expense by performing a (weaker) range check on the + `BigDecimal` representation of incoming `long`s too. + + Relates #26137 + Closes #40323 + +commit 6ef657c5ad60da054b7fa3ec1bd3e7fcb10d0010 +Author: David Kyle +Date: Thu Mar 28 12:27:46 2019 +0000 + + [ML] Data Frame minor tidy ups (#40580) + + Remove Xlint-rawtypes option and remove unused request builders. + Not all requests need to implement ToXContent. + +commit 073b13f5b04b6190aa0d5f2bc5a567da53ef0dc4 +Author: David Turner +Date: Thu Mar 28 12:09:31 2019 +0000 + + Add docs for cluster.remote.*.proxy setting (#40281) + + In #33062 we introduced the `cluster.remote.*.proxy` setting for proxied + connections to remote clusters, but left it deliberately undocumented since it + needed followup work so that it could work with SNI. However, since #32517 is + now closed we can add this documentation and remove the comment about its lack + of documentation. + +commit ddc385d8743fa56269017898cca6b967893f1fb9 +Author: Yannick Welsch +Date: Thu Mar 28 12:12:07 2019 +0100 + + Mute WatchAckTests.testAckAllActions + + Relates to #35506 + +commit 8775e37d03c6b133e5caf863f7ad6bd0a933cb7b +Author: jimczi +Date: Thu Mar 28 09:55:29 2019 +0100 + + Fix SearchResponseMerger#testMergeSearchHits + + This commit fixes an edge case in tests where search hits are empty + after the merge but some shards returned hits. This can happen if + the total number of merged hits is less than the provided `from`. + + Closes #40553 + +commit 89d97905e5bc0907014e61c3dc03322d9d92b93e +Author: Andrei Stefan +Date: Thu Mar 28 10:11:23 2019 +0200 + + SQL: add "fuzziness" option to QUERY and MATCH function predicates (#40529) + + * Remove unused "locale" and "lowercase_expanded_terms" options from QUERY. + + (cherry picked from commit c122fc6edddbb99c73ce25168d1152409c0b7892) + +commit 7f7d09af2ef964f1b7af2d20bcef6c6493f844e1 +Author: Adrien Grand +Date: Thu Mar 28 09:32:26 2019 +0100 + + Deprecate types in `_graph/explore` calls. (#40466) (#40513) + + Any call that uses a path that sets a type will trigger a deprecation warning. + +commit 65a35c985c6f8ed018145f238313e8b4da4d9236 +Author: Adrien Grand +Date: Thu Mar 28 09:32:09 2019 +0100 + + Remove type from VersionConflictEngineException. (#37490) (#40514) + + It initially mentioned the type in the exception because the type used to be + required to uniquely identify a document. This is not necessary anymore given + that indices have at most one type. + +commit 2326a3dccb004269e8c22afc9e5bc11a6408770c +Author: Adrien Grand +Date: Thu Mar 28 09:31:42 2019 +0100 + + Remove String interning from `o.e.index.Index`. (#40350) (#40517) + + `Index` interns its name and uuid. My guess is that the main goal is to avoid + having duplicate strings in the representation of the cluster state. However + I doubt it helps much given that we have many other objects in the cluster state + that we don't try to reuse, and interning has some cost. When looking into + #40263 my profiler pointed to string interning because of the `Index` object + that is created in `QueryShardContext` as one of the bottlenecks of the + `can_match` phase. + +commit 5485efa2af3d8b18ccc8db0f4d0b7bc64f68e390 +Author: Costin Leau +Date: Thu Mar 28 10:06:38 2019 +0200 + + SQL: Polish behavior of SYS TABLES command (#40535) + + SYS TABLES meta command has been improved to better adhere to the ODBC + spec in particular with regards to the handling of enumerations (and + the differences between '%', null and ''(empty string)) + + Fix #40348 + + (cherry picked from commit e3070615000228c283d17ce8d182b44f1450a5d5) + +commit de5f924daac8cdb2f9c4ea01467dbd4b49939a5a +Author: Costin Leau +Date: Wed Mar 27 14:14:06 2019 +0200 + + SQL: Adjust the precision and scale for drivers (#40467) + + Fix #40357 + + (cherry picked from commit 1557d77b07bbeef99604857c1ca824e20771c283) + +commit ebcb925afbd9dd17ec94ae87d5fd6b5aa2a82e55 +Author: Armin Braun +Date: Thu Mar 28 00:57:58 2019 +0100 + + Cleanup Duplication in Netty4 Module (#40148) (#40563) + + * Just drying up the listener/promise handling a little + +commit 5a2ba34174c66bdc484fa907398448ca98f28140 +Author: David Turner +Date: Wed Mar 27 23:08:10 2019 +0000 + + Get node ID from nodes info in REST tests (#40052) (#40532) + + We discussed recently that the cluster state API should be considered + "internal" and therefore our usual cast-iron stability guarantees do not hold + for this API. + + However, there are a good number of REST tests that try to identify the master + node. Today they call `GET /_cluster/state` API and extract the master node ID + from the response. In fact many of these tests just want an arbitary node ID + (or perhaps a data node ID) so an alternative is to call `GET _nodes` or `GET + _nodes/data:true` and obtain a node ID from the keys of the `nodes` map in the + response. + + This change adds the ability for YAML-based REST tests to extract an arbitrary + key from a map so that they can obtain a node ID from the nodes info API + instead of using the master node ID from the cluster state API. + + Relates #40047. + +commit 23395a9b9fd40f83a3fd5b8a5a7f3a05030a6743 +Author: Andy Bristol +Date: Wed Mar 27 13:29:13 2019 -0700 + + search as you type fieldmapper (#35600) + + Adds the search_as_you_type field type that acts like a text field optimized + for as-you-type search completion. It creates a couple subfields that analyze + the indexed terms as shingles, against which full terms are queried, and a + prefix subfield that analyze terms as the largest shingle size used and + edge-ngrams, against which partial terms are queried + + Adds a match_bool_prefix query type that creates a boolean clause of a term + query for each term except the last, for which a boolean clause with a prefix + query is created. + + The match_bool_prefix query is the recommended way of querying a search as you + type field, which will boil down to term queries for each shingle of the input + text on the appropriate shingle field, and the final (possibly partial) term + as a term query on the prefix field. This field type also supports phrase and + phrase prefix queries however + +commit c0c6d702a2638f4ce65bf01ebab44b8515084d6e +Author: Andy Bristol +Date: Wed Mar 27 13:27:17 2019 -0700 + + ignore 409 conflict in reindex responses (#39543) + + The reindex family of APIs (reindex, update-by-query, delete-by-query) can + sometimes return responses that have an error status code (409 Conflict in this + case) but still have a body in the usual BulkByScrollResponse format. When the + HLRC tries to handle such responses, it blows up because it tris to parse it + expecting the error format that errors in general use. This change prompts the + HLRC to parse the response using the expected BulkByScrollResponse format. + +commit 6563dc7ed95c1fc176e6be594338870b7a75c730 +Author: Benjamin Trent +Date: Wed Mar 27 14:52:12 2019 -0500 + + Muting test for #40553 (#40555) + +commit ab44f5fd5d8d3fd9119332af2659dbd0a0fb087c +Author: Tim Brooks +Date: Tue Mar 26 09:32:53 2019 -0600 + + Add InboundHandler for inbound message handling (#40430) + + This commit adds an InboundHandler to handle inbound message processing. + With this commit, this code is moved out of the TcpTransport. + Additionally, finer grained unit tests are added to ensure that the + inbound processing works as expected + +commit 419cf1c02f86ddeb961d791d94e02f7672a84c8e +Author: Julie Tibshirani +Date: Wed Mar 27 11:13:51 2019 -0700 + + Fix an off-by-one error in the vector field dimension limit. (#40489) + + Previously only vectors up to 499 dimensions were accepted, whereas the stated + limit is 500. + +commit 64b31f44af4dc5be776b4d8aea772946fd798425 +Author: Yannick Welsch +Date: Wed Mar 27 18:49:03 2019 +0100 + + No mapper service and index caches for replicated closed indices (#40423) + + Replicated closed indices can't be indexed into or searched, and therefore don't need a shard with + full indexing and search capabilities allocated. We can save on a lot of heap memory for those + indices by not allocating a mapper service and caching infrastructure (which preallocates a constant + amount per instance). Before this change, a 1GB ES instance could host 250 replicated closed + metricbeat indices (each index with one shard). After this change, the same instance can host 7300 + replicated closed metricbeat instances (not that this would be a recommended configuration). Most + of the remaining memory is in the cluster state and the IndexSettings object. + +commit 8f7c5732f19f6b00c607a8e745970c150368df7c +Author: Yannick Welsch +Date: Wed Mar 27 18:48:35 2019 +0100 + + Use default discovery implementation for single-node discovery (#40036) + + Switches "discovery.type: single-node" from using a separate implementation for single-node discovery to using the existing standard discovery implementation, with two small adaptions: + + - auto-bootstrapping, but requiring initial_master_nodes not to be set. + - not actively pinging other nodes using the Peerfinder + - not allowing other nodes to join its single-node cluster (if they have e.g. been set up using regular discovery and connect to the single-disco node). + +commit 3860ddd1a4d970dfc156f56b70f81f14cd631d78 +Author: Tim Brooks +Date: Fri Mar 22 13:58:30 2019 -0600 + + Move outbound message handling to OutboundHandler (#40336) + + Currently there are some components of message serializer and sending + that still occur in TcpTransport. This commit makes it possible to + send a message without the TcpTransport by moving all of the remaining + application logic to the OutboundHandler. Additionally, it adds unit + tests to ensure that this logic works as expected. + +commit 13d4d73ce3ea7613471c9e4a8a5b3fd2531a4b99 +Author: David Kyle +Date: Wed Mar 27 17:36:43 2019 +0000 + + Mute DataFrameTaskFailedStateIT.testFailureStateInteraction (#40544) + +commit 95a0c524a13428ca96273a0897bb09426ada0b86 +Author: Benjamin Trent +Date: Wed Mar 27 12:11:10 2019 -0500 + + Muting test #40368 (#40542) + +commit be67752c34ab692119dd9dc8c9aa59580c45363d +Author: Benjamin Trent +Date: Wed Mar 27 11:47:09 2019 -0500 + + Muting test related to #40537 (#40539) + +commit 8e049c5f583bd0f029b54b9cf1e7b4329633a2d3 +Author: Marios Trivyzas +Date: Wed Mar 27 17:13:18 2019 +0100 + + SQL: Fix getTime() methods in JDBC (#40484) + + Previously, `getTime(colIdx/colLabel)` and + `getObject(colIdx/colLabel, java.sql.Time.class)` methods were computing + the time from a `ZonedDateTime` by applying day in millis modulo on the epoch millis + of the `ZonedDateTime` object. This is wrong as we need to keep the time + related fields at the timezone of the `ZonedDateTime` object and just + set the date info to the epoch date (01/01/1970). + + Additionally fixes a testing issue as the original timezone id is converted + to an offset string when parsing the response from the server. + +commit 707d40ce06813170bad73262ead2e69ba9aba51a +Author: David Turner +Date: Wed Mar 27 15:49:12 2019 +0000 + + Stabilise testStaleMasterNotHijackingMajority (#40253) + + This test inadvertently asserts that the election occurs after a master failure + is clean. However, messy elections are a fact of life so we should not fail on + a messy election. + + This change moves this test away from an `AbstractDisruptionTestCase` since it + does not need the fault detector to be so enthusiastic, and weakens the + assertions to merely say that we ignore states published by the old master + without saying anything about the cleanliness of the election. + + Closes #36556 + +commit f1eca21081af865a2ca84da5fba18c7b0b11507c +Author: David Turner +Date: Wed Mar 27 15:56:02 2019 +0000 + + Mention legacy discovery settings in migration doc (#40528) + + Today the upgrade assistant identifies that discovery configuration is required + in production mode, but links to docs saying to add one of these settings: + + - `discovery.seed_hosts` + - `discovery.seed_providers` + - `cluster.initial_master_nodes` + + However these settings do not exist in 6.7 so this is unhelpful advice. This + commit adjusts the docs to give more useful advice to users arriving at this + page from the upgrade assistant. + + Relates https://discuss.elastic.co/t/174102 + +commit 9a110c793bcdb5392150e09907ae86508e135b74 +Merge: 99dca30197d 59c98fbb66a +Author: James Rodewig +Date: Wed Mar 27 11:39:22 2019 -0400 + + Merge remote-tracking branch 'jrodewig/7.x' into 7.x + +commit 99dca30197de0b9c0877f69dc00f6cb887a00d5d +Author: Andrei Stefan +Date: Wed Mar 27 17:18:14 2019 +0200 + + SQL: MATCH and QUERY documentation; one list of functions (#40494) + + * Document MATCH and QUERY function predicates. + * Polish the functions pages and add a list of functions to the main Functions & Operators page. + + (cherry picked from commit 4cec0ae1b962ec7ea011a290aec72740386eb808) + +commit 760cfffe4b2e5118528037fbf072750b4c54c713 +Author: Tim Brooks +Date: Wed Mar 27 09:24:20 2019 -0600 + + Move TransportMessageListener to TransportService (#40474) + + Currently the TransportMessageListener is applied and used in the + Transport class. However, local requests and responses never make it to + this class. This PR moves the listener add/remove methods to the + TransportService. After this change the Transport can only have one + listener set with it. This one listener is the TransportService, which + will then propogate the events to the external listeners. + + Additionally this commit back ports #40237 + + Remove Tracer from MockTransportService + + Currently the TransportMessageListener is applied and used in the + Transport class. However, local requests and responses never make it to + this class. This PR moves the listener add/remove methods to the + TransportService. After this change the Transport can only have one + listener set with it. This one listener is the TransportService, which + will then propogate the events to the external listeners. + +commit 59c98fbb66af140386376cd93e060bd1f09abc73 +Author: Sébastien Loix +Date: Tue Mar 26 13:37:03 2019 +0100 + + [Docs] Fix command to save SMTP password for email account (#40444) + +commit 03aaeb35cc4a415dc550a7cd3eaeed57a477d357 +Author: James Rodewig +Date: Wed Mar 27 10:31:23 2019 -0400 + + Deprecate elasticsearch.yml as supported Slack config method (#40410) + +commit d57b7a2d5b3fd93edd87bab7bf03524b724414cd +Author: Like +Date: Sat Feb 16 04:50:15 2019 +0800 + + Migrate Streamable to Writeable for WatchStatus (#37390) + +commit 22e688a709fba31268086d3e36bbdf84dcb4a8f5 +Author: Benjamin Trent +Date: Wed Mar 27 09:16:06 2019 -0500 + + Muting watcher tests for issue #35503 (#40527) + + * Muting watcher tests for issue #35503 + + * blocking the two troublsome suites + +commit cfe85c73bf74c7429e7010d17a529f3e44ab1604 +Author: James Rodewig +Date: Wed Mar 27 10:02:37 2019 -0400 + + Fix typo in rollup_index definition (#40520) + +commit 6182b464a36813d1f6b5d82d3b0de6e82d75b06d +Author: Bogdan Pintea +Date: Wed Mar 27 14:11:59 2019 +0100 + + SQL: ODBC: document extra connection string parameters (#40476) + + * document ODBC's extra connection string parameters + + Document the connection string parameters that are currently not + configurable from the GUI. + + Add a note about SmartScreen possible warning on driver MSI + installation. + + Add a note about the TLS certificate file not being supported as bundled + or password-protected. + + * rephrasing and restructuring the list of params + + - addressing PR review notes + + * rephrasings and reference linking + + - addressing PR review notes + + (cherry picked from commit ca90190226c0a8c63f51c2603b27df299ce8d503) + +commit 6efe748e2ee8d56711f3e68fb164207656dfd988 +Author: Boaz Leskes +Date: Wed Mar 27 14:17:29 2019 +0100 + + Only recommend disabling allocation of replicas in a rolling upgrade (#40299) + + The current documentation recommends disabling allocations of all shards. This prevents shards of + new indices from being allocated as well. That means that during a rolling upgrade, operations like + roll over and index shrinking will operate correctly. This, in turn, can cause issues for data ingestion + and ILM. + +commit 61845dd38bc9547666b0a6ddd0832b2725ff8cb2 +Author: David Kyle +Date: Wed Mar 27 12:31:28 2019 +0000 + + [ML] Fix serialisation of Start Data Frame request (#40483) + +commit 65f01277ed2e7787875f4f3a7db34d70616b92ea +Author: Przemyslaw Gomulka +Date: Wed Mar 27 13:51:44 2019 +0100 + + Parse composite patterns using ClassicFormat.parseObject backport(#40100) (#40501) + + Java-time fails parsing composite patterns when first pattern matches only the prefix of the input. It expects pattern in longest to shortest order. Because of this constructing just one DateTimeFormatter with appendOptional is not sufficient. Parsers have to be iterated and if the parsing fails, the next one in order should be used. In order to not degrade performance parsing should not be throw exceptions on failure. Format.parseObject was used as it only returns null when parsing failed and allows to check if full input was read. + + closes #39916 + backport #40100 + +commit c990b30019f50433884eaa25ba759768cb754555 +Author: David Kyle +Date: Wed Mar 27 12:40:39 2019 +0000 + + [ML] Data Frame HLRC Get API (#40509) + +commit b4b17e16e04117064b557a4930e819e6d19551e0 +Author: Yannick Welsch +Date: Wed Mar 27 12:05:53 2019 +0100 + + Remove TransportSingleItemBulkWriteAction as replication action (#40424) + + The implementation of TransportIndexAction and TransportDeleteAction as + TransportReplicationAction existed for interoperability with older 5.x nodes, as these older nodes + coordinated single index / deletes as replication requests. This BWC layer is no longer needed in 7.x, + where these single actions are now mapped to bulk requests. Completely removing the deprecated + transport actions is not possible yet if we want to keep BWC with a 6.x transport client. The best + way here is to wait for the transport client to go away and then just remove the actions. + +commit 856789f7917bc3dd436f6064918c21dbf0d58b0b +Author: Dimitrios Liappis +Date: Wed Mar 27 14:14:47 2019 +0200 + + Wrap Dockerfile yum operations in retries (#40460) + + While yum does retry retrieving files 10 times by default [1], slow + network fetches, governed by `minrate` cause immediate aborts without + getting retried. + + Wrap yum commands in a 10 iteration retry loop. + + [1] http://man7.org/linux/man-pages/man5/yum.conf.5.html + + Backport of #40349 + +commit 12943c5d2cf4a6891bd3f65300cba48392e39777 +Author: Benjamin Trent +Date: Wed Mar 27 06:53:58 2019 -0500 + + [ML] Add data frame task state object and field (#40169) (#40490) + + * [ML] Add data frame task state object and field + + * A new state item is added so that the overall task state can be + accoutned for + * A new FAILED state and reason have been added as well so that failures + can be shown to the user for optional correction + + * Addressing PR comments + + * adjusting after master merge + + * addressing pr comment + + * Adjusting auditor usage with failure state + + * Refactor, renamed state items to task_state and indexer_state + + * Adding todo and removing redundant auditor call + + * Address HLRC changes and PR comment + + * adjusting hlrc IT test + +commit 38dc005b8d90571b5a5bda06378459611404decf +Author: Costin Leau +Date: Wed Mar 27 12:20:16 2019 +0200 + + SQL: Update JDBC class name in client screenshots + + (cherry picked from commit 79a3b93deb858961ecbe4516ed9d8f209016666f) + +commit 33737b6b21fa1cc2a20c23c5208ee7a6c878a329 +Author: Costin Leau +Date: Tue Mar 26 13:38:58 2019 +0200 + + SQL: Polish parsing of CAST expression (#40428) + + + (cherry picked from commit 9d291aa300bbb827eeae606e7d3e55eeef7cce00) + +commit fe05a4d511f866f6effd970dc9dc971eea0df06e +Author: Jim Ferenczi +Date: Wed Mar 27 11:15:25 2019 +0100 + + Fix random failures in SearchResponseMerger#testMergeSearchHits (#40223) + + This commit fixes the expectation in the test when the search hits are empty. + + Closes #40214 + +commit f4e56118c22b833c005ab9f5f9624e22b03e4807 +Author: Hendrik Muhs +Date: Wed Mar 27 08:25:49 2019 +0100 + + [ML] generate unique doc ids for data frame (#40382) + + create and use unique, deterministic document ids based on the grouping values. + + This is a pre-requisite for updating documents as well as preventing duplicates after a hard failure during indexing. + +commit 524e0273aeacac8a3865fa3f0bf1b25caf152a25 +Author: Alpar Torok +Date: Wed Mar 27 08:37:04 2019 +0200 + + Testclusters: convert plugin repository-s3 (#40399) + + * Add support for setting and keystore settings + * system properties and env var config + * use testclusters for repository-s3 + * Some cleanup of the build.gradle file for plugin-s3 + * add runner {} to rest integ test task + +commit aa24669e956cd8921369275abd3fae3cf901c0f2 +Author: Ryan Ernst +Date: Tue Mar 26 19:48:02 2019 -0700 + + Resolve JAVA_HOME at windows service install time (#39714) + + On windows, JAVA_HOME is currently resolved when the windows service is started. However, this is contrary to what our documentation states. This commit moves resolution to service install. This has the side effect of making java existence checking optional in elasticsearch-env.bat, since the rest of the service commands do not require java. + + closes #30720 + +commit f82f9423b7dc905da3a3c9380363013caae6fd4b +Author: Ryan Ernst +Date: Tue Mar 26 19:46:58 2019 -0700 + + Remove platform test command (#40280) + + The platformTest gradle task was a packaging test meant to ensure unit + tests run on the various supported operating systems without relying on + CI to maintain a full matrix of platforms. Howevever, it never really + worked out as intended and is now additional code in our vagrant setup + to maintain. This commit removes the platformTest task. + +commit 81510746d635cac6d935edce2022a13f92bfa81f +Author: lcawl +Date: Tue Mar 26 15:45:53 2019 -0700 + + [DOCS] Adds anchors and x-pack icons for frozen indices + +commit cbdb20faba7c8150b3ab4587eec99b94b09e35ea +Author: Jason Tedor +Date: Tue Mar 26 18:47:57 2019 -0400 + + Add more Java 12 distributions to testing (#40482) + + This commit adds the Oracle JDK 12 and the Zulu JDK 12 distributions to + testing. + +commit 25954a8dd316851e387b5a24b3ea69fdd1c97fbb +Author: Julie Tibshirani +Date: Tue Mar 26 13:08:08 2019 -0700 + + Stop clearing all watches in watcher integration tests. (#39724) + +commit 562ebaefa6b50f70592ce4e3d9b42e3159cfcdd4 +Author: Ryan Ernst +Date: Tue Mar 26 11:41:22 2019 -0700 + + Delete jdk extract dir before extracting (#40471) + + This commit first deletes the directory we extract the bundled jdk into in order to make gradle's copy task happy. + +commit e49ba6a9d9deaeefcb942671d7dc3684e4bb08fc +Author: Ryan Ernst +Date: Tue Mar 26 11:26:01 2019 -0700 + + Remove unused variable (#40468) + + This was leftover from previous iterations of test clusters refactorings + +commit fb8ad0cf30ddacf863f2978500679421690fd8e7 +Author: alex101101 +Date: Wed Mar 27 00:29:24 2019 +0800 + + Add a soft limit to the field name length (#40309) + + Adds an optional limit to the length of field names, throws an IllegalArgumentException if the limit is breached. + Closes #33651 + +commit f2b5960f906d047efa315ae5f8fd27f808ace8f4 +Author: Daniel Mitterdorfer +Date: Tue Mar 26 17:37:35 2019 +0100 + + Add version 6.7.1 + +commit 324a5400a4ca1dd6e7210b5dd0874686566376ac +Author: debadair +Date: Tue Mar 26 09:19:44 2019 -0700 + + [DOCS] Switched release state back to unreleased. Closes #40259 (#40470) + +commit bf7b167bbadffa179d9bda0bbb02409b8f27e688 +Author: Yannick Welsch +Date: Tue Mar 26 17:00:28 2019 +0100 + + Remove timeout task after completing cluster state publication (#40411) + + Each cluster state publication schedules a cancellation task with the provided publication timeout + (30s by default). This scheduled cancellation keeps a reference to the publication, and therefore the + full cluster state that was published. In case of frequently updating a large cluster state, this results + in a large number of cancellation tasks keeping references to all previously published cluster states. + +commit 9bd8600c2e4f86180d06c2741ca2e715c9acc793 +Author: Jay Modi +Date: Tue Mar 26 08:40:00 2019 -0600 + + Use ephemeral ports for idp-fixture (#40333) + + This change removes the use of hardcoded port values for the + idp-fixture in favor of the mapped ephemeral ports. This should prevent + failures due to port conflicts in CI. + +commit bf444b9f02d3517c0fdefd38a1329472b963e6f6 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Mar 26 15:11:51 2019 +0100 + + Store Pending Deletions Fix (#40345) + + FilterDirectory.getPendingDeletions does not delegate, fixed + temporarily by overriding in StoreDirectory. + + This in turn caused duplicate file name use after a trimUnsafeCommits + had been done, since a new IndexWriter would not consider the pending + deletes in IndexFileDeleter. This should only happen on windows (AFAIK). + + Reenabled doing index updates for all tests using + IndexShardTests.indexOnReplicaWithGaps (which could fail due to above + when using mocked WindowsFS). + + Added getPendingDeletions delegation to all elasticsearch + FilterDirectory subclasses that were not trivial test-only overrides to + minimize the risk of hitting this issue in another case. + +commit b5a8de9a7f715762d70081db6510e8cb0c300ec5 +Author: Alpar Torok +Date: Tue Mar 26 13:27:01 2019 +0200 + + Revert tmpdir (#40312) + + * Revert "Configure TMP for test nodes on Windows (#39959)" + This reverts commit 97562a874fcb1f29fb05272ab860a0307e97d1aa. + + * Configure a tmp dir without spaces + * Pass on TMP instead of changing it + +commit d50305bbee7d4dce1b8b5f11c14676d64620000c +Author: Alpar Torok +Date: Tue Mar 26 13:20:04 2019 +0200 + + Upgrade to Gradle 5.3! (#40346) + + Here are the highlights of this release: + - Feature variants AKA "optional dependencies" + - Type-safe accessors in Kotlin precompiled script plugins + - Gradle Module Metadata 1.0 + + For more details see https://docs.gradle.org/5.3/release-notes.html + +commit 1354696db9d91dbccdbe2a8377609aaae356b0d8 +Author: David Kyle +Date: Tue Mar 26 11:17:13 2019 +0000 + + [ML] Data Frame HLRC Get Stats API (#40443) + +commit cd4eac5fa41ee56cf2623f5d275c64325f7534a3 +Author: Alpar Torok +Date: Tue Mar 26 13:06:45 2019 +0200 + + Testclusters links (#39831) + + * Create hardlinks instead of sync to save on disk space + +commit 2b34923541de5acffd479f188bada497d3eb523d +Author: Alpar Torok +Date: Tue Mar 26 12:18:21 2019 +0200 + + Add Gradle init script to configure artifactoy (#40412) + + The script will be use in CI to configure our own Artifactory instance + so CI does not depend on external network resources nor do we generate + excessive load on these resources. + In case the repo is not accessible Gradle should fall back to using the + public ones. + +commit 12634850d6523fbb9f0fcf7e008d2c4121da346a +Author: Alan Woodward +Date: Tue Mar 26 08:11:39 2019 +0000 + + IntervalQueryBuilderTests#testNonIndexedFields test fix (#40418) + + This test checks that interval queries constructed against a field with no indexed + positions will throw exceptions. It uses a randomly-build IntervalsSourceProvider + against a fixed set of fields; however, the random source builder can occasionally + provide a source with a fixed field, meaning that even if the top-level query asks + for a set of intervals over a non-indexed field, the source will delegate to another + field, and no exception will be thrown. + + This commit changes the test to always use a simple Match provider. + + Fixes #40436 + +commit cafb83297c2c7b4c0f4bd85ec0ba6aa222120cc3 +Author: Armin Braun +Date: Tue Mar 26 09:04:15 2019 +0100 + + Use Correct Enum in Wipe Snapshots Test Method (#40422) (#40438) + + * Mistake was made in #39662 + * The response deserialized here is `org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse` which uses `org.elasticsearch.snapshots.SnapshotInfo` which uses `org.elasticsearch.snapshots.SnapshotState` and not the shard state + +commit 1d3ece1e96fe196ecd68e896f3d638c196b0eedd +Author: Martijn van Groningen +Date: Tue Mar 26 07:54:22 2019 +0100 + + Remove -Xlint exclusions in the percolator module. (#40372) + + Relates to #40366 + +commit 7234d78747d50805a9cf912e88d1a2427ab399f5 +Author: Costin Leau +Date: Tue Mar 26 08:17:08 2019 +0200 + + SQL: Fix classpath discovery on Java 10+ (#40420) + + + (cherry picked from commit 2cef233cb34ee80d8ed9cd014cea76ea5096d206) + +commit c20ea9a2dd847f1d5a29ad6d6774c1241850109d +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Mon Mar 25 11:55:04 2019 +0000 + + [ML][TEST] Fix failing test testPersistJobOnGracefulShutdown_givenTimeAdvancedAfterNoNewData (#40363) + + Ensure that there is at least a 1s delay between the time that state + is persisted by each of the two jobs in the test. + + Model snapshot IDs use the current time in epoch seconds to + distinguish themselves, hence snapshots will be overwritten + by another if it occurs in the same 1s window. + + Closes #40347 + +commit 6f03a6c54656a08112168e8a3b29bac1f9cd9037 +Author: Lisa Cawley +Date: Mon Mar 25 10:23:10 2019 -0700 + + [DOCS] Simplify ML upgrade step (#40006) + +commit 495dc11c9ce8acef13a3c4947d43b92c6409b794 +Author: Nhat Nguyen +Date: Mon Mar 25 11:17:07 2019 -0400 + + Mute testPendingRefreshWithIntervalChange + + Tracked at #39565 + +commit 3968d46a171758fa66d9fd73806fc47f61282cf8 +Author: Armin Braun +Date: Mon Mar 25 16:36:02 2019 +0100 + + Remove Redundant Request Wrappers from RepositoryService (#40192) (#40404) + +commit 61f49af497f51a54792f6aaa999da938d6a36678 +Author: Costin Leau +Date: Mon Mar 25 15:22:59 2019 +0200 + + SQL: Spec tests now use classpath discovery (#40388) + + To avoid having to specify each spec by hand (which can miss specs to be + added), the test infrastructure now performs classpath discovery so that + each spec added, is automatically considered. + + Relates #40358 + + (cherry picked from commit d0f60b4425c731509aa8ca765d55f563f866ef90) + +commit e8a93b8915019acdf3a296dede6cb0f67384346b +Author: James Rodewig +Date: Mon Mar 25 09:00:03 2019 -0400 + + Update Windows command to support prettified json licenses (#40407) + +commit 7b4f964708a038f8e90b6210a17a3a85eb8e9aa6 +Author: Benjamin Trent +Date: Mon Mar 25 07:16:41 2019 -0500 + + [ML] make source and dest objects in the transform config (#40337) (#40396) + + * [ML] make source and dest objects in the transform config + + * addressing PR comments + + * Fixing compilation post merge + + * adding comment for Arrays.hashCode + + * addressing changes for moving dest to object + + * fixing data_frame yml tests + + * fixing API test + +commit 38afc9f27de580d86179553b73001b76c20b82fe +Author: Hendrik Muhs +Date: Mon Mar 25 11:57:11 2019 +0100 + + refresh audit index before searching (#40401) + + refresh the audit index before searching + +commit dc5ff0fffcce3574491086c774fefc31a3fdd80f +Author: Armin Braun +Date: Mon Mar 25 08:52:09 2019 +0100 + + Log Warning on Failed Blob Deletes in BlobStoreRepository (#40188) (#40340) + + * Log Warning on Failed Blob Deletes in BlobStoreRepository + * We should not just debug log these spots, they all can and will lead to leaked files when snapshot deletion fails + +commit b9f96a8e1f2e6760379ad18c38bee174b34362ef +Author: Nhat Nguyen +Date: Sun Mar 24 22:21:00 2019 -0400 + + Expose external refreshes through the stats API (#38643) + + Right now, the stats API only provides refresh metrics regarding + internal refreshes. This isn't very useful and somewhat misleading for + cluster administrators since the internal refreshes are not indicative + of documents being available for search. + + In this PR I added a new metric for collecting external refreshes as + they occur and exposing them through the stats API. Now, calling an + endpoint for stats will yield external refresh metrics as well. + + Relates #36712 + +commit 13d76239a0f4626b9cb02d7867c405408a39c202 +Author: Armin Braun +Date: Sun Mar 24 19:08:51 2019 +0100 + + Use Netty ByteBuf Bulk Operations for Faster Deserialization (#40158) (#40339) + + * Use bulk methods to read numbers faster from byte buffers + +commit a30bf27b2f629371033d30ef1efc193364b56bbe +Author: Benjamin Trent +Date: Sat Mar 23 18:56:44 2019 -0500 + + [ML] add auditor to data frame plugin (#40012) (#40394) + + * [Data Frame] add auditor + + * Adjusting Level, Auditor, and message to address pr comments + + * Addressing PR comments + +commit 2dd879abacecf9ba29cea47040133a3ecc5523e9 +Author: Benjamin Trent +Date: Sat Mar 23 14:04:14 2019 -0500 + + [ML] adds support for non-numeric mapped types (#40220) (#40380) + + * [ML] adds support for non-numeric mapped types and mapping overrides + + * correcting hlrc compilation issues after merge + + * removing mapping_override option + + * clearing up unnecessary changes + +commit 88f510ffc2a9bc4fe4301d32a7a1f6127abebf56 +Author: Benjamin Trent +Date: Sat Mar 23 12:15:37 2019 -0500 + + [ML] making test more determinate (#40374) (#40381) + + * [ML] making test more determinate + + * unmuting test + +commit 03839ba1a2bb6c72f32e0e14f71b0dae6a038ebf +Author: Jason Tedor +Date: Sat Mar 23 12:55:48 2019 -0400 + + Update feature aware check ASM to 7.1 (#40389) + + This commit updates the feature aware check ASM dependency to ASM + 7.1. This gives us JDK 13 compatibility. + +commit 143db10980c6e9ce6be3d4dd5a5b5b67e7f9e5ed +Author: Marios Trivyzas +Date: Sat Mar 23 16:59:19 2019 +0100 + + SQL: Fix issue timezone issues with JDBC getDate/getTime (#40360) + + Previously, `getDate(int columnIdx)/getDate(String columnLabel)` and + were using legacy`java.util.Calendar` instead of the the `java.time.*` + classes to reset to the start of day. This resulted in different results + for certain timestamps and timezones when calling + `getDate(col)` vs`getObject(col, java.sql.Date)` + + Now only the methods (that must be implemented due to the JDBC spec) + `getDate(int columnIdx, Calendar cal)/getDate(String columnLabel, Calendar cal)` + are still using the `java.util.Calendar` for those conversion. + + The same change was applied to + `getTime(int columnIdx)/getTime(String columnLabel)` + and + `getTimestamp(int columnIdx)/getTimestamp(String columnLabel)` + + Fixes: #40289 + + (cherry picked from commit 44560671f18397e0c58e3647732880fcb73a5034) + +commit 10bbb082a4414831b80fc4e12328299734a5495f +Author: Jason Tedor +Date: Sat Mar 23 09:38:49 2019 -0400 + + Only run retention lease actions on active primary (#40386) + + In some cases, a request to perform a retention lease action can arrive + on a primary shard before it is active. In this case, the primary shard + would not yet be in primary mode, tripping an assertion in the + replication tracker. Instead, we should not attempt to perform such + actions on an initializing shard. This commit addresses this by not + returning the primary shard in the single shard iterator if the primary + shard is not yet active. + +commit 17b8b54d5e11c2edc0beeee7db790c6129a7aa91 +Author: Marios Trivyzas +Date: Sat Mar 23 14:11:32 2019 +0100 + + SQL: Fix metric aggs on date/time to not return double (#40377) + + Previously metric aggregations on date fields would return a double + which caused errors when trying to apply scalar functions on top, e.g.: + ``` + SELECT YEAR(MAX(date)) FROM test + ``` + + Fixes: #40376 + + (cherry-picked from commit 41d0a038467fbdbbf67fd9bfdf27623451cae63a) + +commit 558adc0f28c96a0f11cead78840cc57d604ba657 +Author: Costin Leau +Date: Sat Mar 23 12:41:51 2019 +0200 + + SQL: Add missing handling of IP field in JDBC (#40384) + + Fix #40358 + + (cherry picked from commit ee286fa4893817637c05d72b93b254b36efc0dae) + (cherry picked from commit d2296249499e31bd512390ac3d20bc38009612b3) + +commit 150d1332cf56337a1187fa01e6922d475a79f40a +Author: Andrei Stefan +Date: Sat Mar 23 06:36:44 2019 +0200 + + SQL: Fix RLIKE bug and improve testing for RLIKE statement (#40354) + + * Refactor RegexMatch to support both LIKE and RLIKE + * Add integration tests for RLIKE + * Polish the rest of tests + + (cherry picked from commit 7562d6eeeb77c04794002649fe726f4b3a9a398b) + +commit 421d7654a7f24a584a51e1d222284f5b71d77849 +Author: lcawl +Date: Fri Mar 22 16:37:03 2019 -0700 + + [DOCS] Adds 7.1 breaking changes + +commit 87d3d16c5ad6df20cc0fd69a7e3c82cccd4f3d8c +Author: Costin Leau +Date: Fri Mar 22 23:20:41 2019 +0200 + + SQL: JLine upgrade and polishing (#40321) + + Upgrade JLine to 3.10.0 + Switch to using JLine granular jars instead of the uber-one + Remove Jansi dependency (due to errors in closing streams) + Pin JNA dependency to our own artifact + + Fix #40239 + + (cherry picked from commit 9afa65fa80111f3b68c13373c7b6db13c11dde31) + +commit 496070fda69f66a857a1810cc3fa06905dd954e3 +Author: Costin Leau +Date: Fri Mar 22 20:42:54 2019 +0200 + + SQL: CAST supports both SQL and ES types (#40365) + + Extend CAST to support all data types notations (whether SQL or ES + specific) + + Fix #40282 + + (cherry picked from commit eb2ee8a344da946920598839a5db76c8bb9bc3fe) + +commit eb10afabb858573aa1a764d0ddc2b12da8249ac6 +Author: lcawl +Date: Fri Mar 22 14:40:59 2019 -0700 + + [DOCS] Fixes comments in release highlights + +commit bc8c9f358d94e09eada318b704ff1bbea252be67 +Author: lcawl +Date: Fri Mar 22 13:36:37 2019 -0700 + + [DOCS] Adds 7.1.0 release highlights + +commit 0be7780cb0a693bd274c91df3e4a61720794871b +Author: Jack Conradson +Date: Fri Mar 22 12:54:26 2019 -0700 + + Add implicit this for class binding in Painless (#40285) + + This change allows class bindings to add as their first argument, the base script + class. The this reference to the base script class will be implicitly passed into a + class binding as the first constructor argument upon initialization when + specified as the first argument in whitelist entry for the class binding. This + allows a class binding access to additional information added to the base script + class such as more information about the current document or current shard. + One extra requirement for this to work is the appropriate script base class + must be whitelisted (should be empty). + +commit 15a1e65e481c0980b5c990a31eb86ea56ece145b +Author: lcawl +Date: Fri Mar 22 11:24:22 2019 -0700 + + [DOCS] Adds placeholder for 7.0.0-rc1 release notes + +commit 05460cca58f9bc98d1d4ad2312ad25fce3c90fae +Author: Benjamin Trent +Date: Fri Mar 22 13:25:48 2019 -0500 + + Muting test testExtractIndexCheckpointsInconsistentGlobalCheckpoints (#40370) + +commit 970cd57368a945bc6a5fa11d312faf7a3d2f2f68 +Author: Costin Leau +Date: Fri Mar 22 19:40:14 2019 +0200 + + DOC: polish client docs + + (cherry picked from commit 72e413df26791b90475ce2f22c31d29e79bd52b9) + +commit b52d44cc9891d068d9ba8298d980f4a701fe899b +Author: Alpar Torok +Date: Fri Mar 22 19:27:01 2019 +0200 + + Fix building bwc versions (#40361) + + Account for openjdk when figuring out what java version to + use for building bwc versions. + +commit 9acd20874b5594298ecb075fd582303e43ead0bc +Author: Jason Tedor +Date: Fri Mar 22 08:50:13 2019 -0400 + + Update contributing docs to JDK 12 + + JDK 12 is now required for compilation. This commit updates the docs to + reflect this. + +commit 3212557da4f3da442a35d58ab80a3fbf7eea4983 +Author: Jason Tedor +Date: Fri Mar 22 08:32:45 2019 -0400 + + Require JDK 12 for compilation (#40231) + + This commit switches from requiring JDK 11 for compilation to requiring + JDK 12 for compilation. + +commit 5a0c32833ecf775758d481198335c83f2877cce2 +Author: Hendrik Muhs +Date: Fri Mar 22 09:56:41 2019 +0100 + + Add a checkpoint service for data frame transforms (#39836) + + Add a checkpoint service for data frame transforms, which allows to ask for a checkpoint of the + source. In future these checkpoints will be stored in the internal index to + + - detect upstream changes + - updating the data frame without a full re-run + - allow data frame clients to checkpoint themselves + +commit 1265a15b75fe8ce44fb426acd802b65fb2201506 +Author: David Turner +Date: Fri Mar 22 08:18:37 2019 +0000 + + Mute testPersistJobOnGracefulShutdown_givenTimeAdvancedAfterNoNewData + +commit 980ee14f5704d5d48cf6ac1e3d3095f3c4e3cff6 +Author: Costin Leau +Date: Fri Mar 22 10:03:42 2019 +0200 + + DOC: Expand section on ORDER BY aggs (#40332) + + + (cherry picked from commit 99d2f6fc9864ab972259ef5692129ab49e4a7ab8) + +commit f9ab9afcc1d85ab77bfc10d46154964805cfc603 +Author: Andrei Stefan +Date: Thu Mar 21 21:31:47 2019 +0200 + + Extract the first value in an array when looking at the returned values (#40318) + + (cherry picked from commit faf02e0f42a101985619abc0d30753851605e01d) + +commit 35fe05308e4485da3fee9f641e78de70ebf4d184 +Author: Andrei Stefan +Date: Thu Mar 21 21:28:31 2019 +0200 + + SQL: rewrite ROUND and TRUNCATE functions with a different optional parameter handling method (#40242) + + * Rewrite Round and Truncate functions to have a slightly different + approach to handling the optional parameter in the constructor. Until now + the optional parameter was considered 0 if the value was missing and the + constructor was filling in this value. The current solution is to have + the optional parameter as null right until the actual calculation is done. + + (cherry picked from commit 3e314f8fa4cb322e67949e80857561ce51268726) + +commit 280567da8d7feb17caa3693e965e666e4da669f9 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Mar 22 12:38:03 2019 +1100 + + Correct documentation link for authorization engine example (#40261) (#40292) + + This commit fixes the link for authorization engine example. + +commit 0e12065b5472c5919f20a59c937d283cf29fdf81 +Author: Nhat Nguyen +Date: Thu Mar 21 18:05:08 2019 -0400 + + Relax max_seq_no_of_updates assertion in follow tests + + If there's a failover on the follower, then its max_seq_no_of_updates is + bootstrapped from its max_seq_no which might be higher than the + max_seq_no_of_updates of the leader. We need to relax this check. + + Relates #40249 + +commit 6ea3272f41f13e5af8f03bcea0ad136c8fdc6b39 +Author: Jack Conradson +Date: Thu Mar 21 16:07:49 2019 -0700 + + Add double and Double standard casts tests to Painless (#40324) + +commit b71066129bf4c22a472abc7a58eabce248fe9edc +Author: Tim Brooks +Date: Thu Mar 21 12:08:23 2019 -0600 + + Expand `following` documentation in ccr overview (#39936) + + This commit expands the ccr overview page to include more information + about the lifecycle of following an index. It adds information linking + to the remote recovery documentation. And describes how an index can + fall-behind and how to fix it when this happens. + +commit 78f737dad32e2816b607ee0373ed8a756ad2efe2 +Author: Zachary Tong +Date: Thu Mar 21 14:03:14 2019 -0400 + + Map value field to double in MovavgIT (#40230) + + We were accidentally not mapping the index, which meant dynamic mapping + was choosing floats for the values. This led to enough loss of precision + for the aggregated values to differ slightly from the test doubles, + which accumulated into large differences in the holt output. + + This test fix adds an explicit mapping. + +commit 23d5f7babf42397172778e69c9d02602e99487c2 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Thu Mar 21 17:01:10 2019 +0000 + + [ML] Add integration tests to check persistence (#40272) (#40315) + + Additional checks to exercise the behaviour of + persistence on graceful close of an anomaly job. + + Related to elastic/ml-cpp#393 + Backports #40272 + +commit e6799849d1ad2b4b4c7ba8a71ea354ded9a54340 +Author: Lisa Cawley +Date: Thu Mar 21 09:38:00 2019 -0700 + + [DOCS] Adds placeholder for start and stop data frame transform APIs (#40278) + +commit caa0129d447957a744eea218432351c7797f3237 +Author: Lisa Cawley +Date: Thu Mar 21 09:11:38 2019 -0700 + + [DOCS] Adds placeholder for create and delete data frame transform APIs (#40233) + +commit 0e712d476e78d88504e9c148d797b9888dbc1385 +Author: lcawl +Date: Thu Mar 21 08:27:43 2019 -0700 + + Adds URL for preview data frame transforms + +commit f4b819a578540d6f970be188f4006f491043f696 +Author: RomainGeffraye +Date: Thu Mar 21 15:54:11 2019 +0100 + + Fix typo (#40273) + +commit ff2bcc9d11e7ef9b6b3a8cf086a54aa35c03b535 +Author: Lisa Cawley +Date: Thu Mar 21 07:45:17 2019 -0700 + + [DOCS] Adds placeholder for get data frame transform APIs (#40283) + +commit 2f80b7304f756426c1d32b88d3f087db8b9c7923 +Author: Albert Zaharovits +Date: Thu Mar 21 15:55:56 2019 +0200 + + Refactor Token Service (#39808) + + This refactoring is in the context of the work related to moving security + tokens to a new index. In that regard, the Token Service has to work with + token documents stored in any of the two indices, albeit only as a transient + situation. I reckoned the added complexity as unmanageable, + hence this refactoring. + + This is incomplete, as it fails to address the goal of minimizing .security accesses, + but I have stopped because otherwise it would've become a full blown rewrite + (if not already). I will follow-up with more targeted PRs. + + In addition to being a true refactoring, some 400 errors moved to 500. Furthermore, + more stringed validation of various return result, has been implemented, notably the + one of the token document creation. + +commit 244e6758cd766b45df51453d7bb35a8248e5d277 +Author: James Rodewig +Date: Thu Mar 21 08:58:26 2019 -0400 + + [DOCS] Add 'recovery' to glossary for issue #7264 (#40275) (#40311) + +commit dd41ce076360034be2adb10dd6a55e3aae91d1bb +Author: Costin Leau +Date: Thu Mar 21 14:08:16 2019 +0200 + + SQL: Preserve original source for cast/convert function (#40271) + + Improve rule for pruning cast to preserve the original source + Fix #40239 + + (cherry picked from commit 7591cb1a1577320b3aec2ec557b0f881b6af744f) + +commit 1e6941b138cbd73a854201ba54a984f549e0402b +Author: Jason Tedor +Date: Thu Mar 21 07:37:04 2019 -0400 + + Reduce retention lease sync intervals (#40302) + + This commit adjusts the frequency with which CCR renews retention leases + and with which primaries sync retention leases to replicas. This helps + Lucene reclaim soft-deleted documents more aggressively, which we have + found in some use-cases can help improve performance, and either way + will help keep disk space under more control. + +commit 1a5ff058706ee737070ffa1a1c5d7b6c10b0b384 +Author: Andrei Stefan +Date: Thu Mar 21 11:42:18 2019 +0200 + + SQL: fix LIKE function equality by considering its pattern as well (#40260) + + * Define a equals method for Like function so that the pattern used + is considered in the equality check. Whenever the functions are resolved + this check should be used. + + (cherry picked from commit 4e5d5af58a140573b8ee19d57c7839db7b779e3b) + +commit a4cb92a3003dcd59e83d3e67f2a5174162670dc3 +Author: David Kyle +Date: Thu Mar 21 09:38:27 2019 +0000 + + [ML] Data Frame HLRC Preview API (#40258) + +commit d485be631b9164b66a65757f38e9971c2e055eec +Author: Andrei Stefan +Date: Thu Mar 21 10:56:29 2019 +0200 + + Moving tests in locale-aware test file (#40254) + + (cherry picked from commit 9beb31fd3c5a8323cb08cc524f1a2268e9c72c24) + +commit 5d30df5a6056479b17037caa29c07db8c1459651 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Mar 21 15:53:25 2019 +1100 + + Fix so non super users can also create API keys (#40028) (#40286) + + When creating API keys we check for if API key with + the same key name already exists and fail the request if it does. + The check should have been performed with XPackSecurityUser + instead of the authenticated user. This caused the request to fail + in case of the non-super user trying to create an API key. + This commit fixes by executing search action with SECURITY_ORIGIN + so it can be executed with XPackSecurityUser. + Also fixed the Rest test to avoid using a user with `super_user` role. + + Closes #40029 + +commit e1eb683c51ff8bcd09613543e7faf1cdfae9d509 +Author: Marios Trivyzas +Date: Thu Mar 21 01:14:07 2019 +0100 + + SQL: Fix issue with getting DATE type in JDBC (#40207) + + Previously, calling getDate()/getTime()/getTimestamp() and getObject() + with the corresponding java.sql class on a column of SQL DATE type from + the JDBC result set would throw an Exception. + +commit 5ae43855fc372dc15621e5b873f628ab0a0e18d0 +Author: Benjamin Trent +Date: Wed Mar 20 19:14:50 2019 -0500 + + [ML] Refactor GET Transforms API (#40015) (#40269) + + * [Data Frame] Refactor GET Transforms API: + + * Add pagination + * comma delimited list expression support GET transforms + * Flag troublesome internal code for future refactor + + * Removing `allow_no_transforms` param, ratcheting down pageparam option + + * Changing DataFrameFeatureSet#usage to not get all configs + + * Intermediate commit + + * Writing test for batch data gatherer + + * Removing unused import + + * removing bad println used for debugging + + * Updating BatchedDataIterator comments and query + + * addressing pr comments + + * disallow null scrollId to cause stackoverflow + +commit f37f2b5d39676d45706efd29b2fab9650158d5a7 +Author: Marios Trivyzas +Date: Wed Mar 20 23:51:15 2019 +0100 + + SQL: Fix issue with optimization on queries with ORDER BY/LIMIT (#40256) + + Previously, when a trival plain `SELECT` or a trivial `SELECT` with + aggregations has also an `ORDER BY` or a `LIMIT` or both, then the + optimization to convert it to a `LocalRelation` was skipped resulting + in exception thrown. E.g.:: + ``` + SELECT 'foo' FROM test LIMIT 10 + ``` + or + ``` + SELECT 'foo' FROM test GROUP BY 1 ORDER BY 1 + ``` + + Fixes: #40211 + +commit bc4c8e53c5034d34d5322ac126ce8bcfc385f3cd +Author: Marios Trivyzas +Date: Wed Mar 20 19:58:57 2019 +0100 + + SQL: Fix issue with date columns returned always in UTC (#40163) + + When selecting columns of ES type `date` (SQL's DATETIME) the + `FieldHitExtractor` was not using the timezone of the client session + but always resorted to UTC. The same behaviour (UTC only) was + encountered also for grouping keys (`CompositeKeyExtractor`) and + for First/Last functions on dates (`TopHitsAggExtractor`). + + Fixes: #40152 + +commit 95f1c8bea0a375d2d4d4a2eec6ff5222a8f164a7 +Author: James Rodewig +Date: Wed Mar 20 15:09:36 2019 -0400 + + Add links to Freeze/Unfreeze index APIs (#40225) (#40268) + +commit 75a788ec5d30237cba04ac126d4992505cedfd74 +Author: Lisa Cawley +Date: Wed Mar 20 10:19:41 2019 -0700 + + [DOCS] Adds placeholder for data frame preview API (#40232) + +commit 606411bce8daed8b56d6c188ef4315f69dc87efe +Author: Nhat Nguyen +Date: Wed Mar 20 12:43:31 2019 -0400 + + Mute testRelocationWithConcurrentIndexing + + Tracked at #34950 + +commit 83d2870308ebc512f0524c6d5d61da1863c7dac3 +Author: Alan Woodward +Date: Wed Mar 20 16:25:15 2019 +0000 + + Add `use_field` option to intervals query (#40157) + + This is the equivalent of the `field_masking_span` query, allowing users to + merge intervals from multiple fields - for example, to search for stemmed tokens + near unstemmed tokens. + +commit 6f6426762601aa22d06663ebc57c0e572c6321af +Author: Like +Date: Wed Mar 20 23:55:59 2019 +0800 + + Make setting index.translog.sync_interval be dynamic (#37382) + + Currently, we cannot update index setting index.translog.sync_interval if index is open, because it's + not dynamic which can be updated for closed index only. + + Closes #32763 + +commit 5ec56d7d229bea9f40f63ae23962549b76f7cf3a +Author: Jack Conradson +Date: Wed Mar 20 08:52:58 2019 -0700 + + Add float and Float standard casting tests to Painless. (#40221) + +commit a5fb7fb17c78dd59dd51de34e809cb663cd49801 +Author: Yannick Welsch +Date: Wed Mar 20 16:51:04 2019 +0100 + + Fix snapshot restore logging on fresh restore (#40252) + + A recent refactoring (#37130) where imports got mixed up (changing Lucene's + IndexNotFoundException to Elasticsearch's IndexNotFoundException) led to many warnings being + logged in case of restoring a fresh snapshot. + +commit 3400483af46189435e6039806dbee6b8fd9a8532 +Author: Jim Ferenczi +Date: Wed Mar 20 16:50:28 2019 +0100 + + Add date and date_nanos conversion to the numeric_type sort option (#40199) (#40224) + + This change adds an option to convert a `date` field to nanoseconds resolution + and a `date_nanos` field to millisecond resolution when sorting. + The resolution of the sort can be set using the `numeric_type` option of the + field sort builder. The conversion is done at the shard level and is restricted + to dates from 1970 to 2262 for the nanoseconds resolution in order to avoid + numeric overflow. + +commit 5eb33f2df4f4331778f058f238a81ca901806fc9 +Author: David Turner +Date: Wed Mar 20 15:49:50 2019 +0000 + + Document the reduction in fault detection timeouts (#40200) + + The new cluster coordination subsystem introduced in 7.0 will only keep an + unresponsive node in the cluster for 30 seconds, whereas in earlier versions it + might have remained in the cluster for 90 seconds. This commit adds a note to + the migration documentation to that effect. + +commit 8f84f455c37577ea6a1ae2e1758d45386551b02f +Author: Yannick Welsch +Date: Wed Mar 20 16:31:03 2019 +0100 + + Remove note about Azure ARM plugin (#40219) + + Relates to #22679 + +commit efaf95628b0ea6d116d5c0327f20f4233cf63698 +Author: Nhat Nguyen +Date: Wed Mar 20 10:19:34 2019 -0400 + + Use separate translog dir in testDeleteWithFatalError + + This test currently opens a new engine but shares the same translog + directory of the previous opening engine. + +commit 49a7c6e0e8d113ba76d1b2667cfcb3617396191e +Author: Mayya Sharipova +Date: Wed Mar 20 09:24:41 2019 -0400 + + Expose proximity boosting (#39385) (#40251) + + Expose DistanceFeatureQuery for geo, date and date_nanos types + + Closes #33382 + +commit 4c2a8638caf040cc4b328f8a53848db40fc3e451 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Mar 20 13:56:38 2019 +0100 + + Cascading primary failure lead to MSU too low (#40249) + + If a replica were first reset due to one primary failover and then + promoted (before resync completes), its MSU would not include changes + since global checkpoint, leading to errors during translog replay. + + Fixed by re-initializing MSU before restoring local history. + +commit 68d98c85c9f2cd3e961489fddf31b01423b1b1c6 +Author: Jason Tedor +Date: Wed Mar 20 08:52:59 2019 -0400 + + Use bundled JDK in Docker images (#40238) + + Now that we have the bundled JDK in the Docker images, we should use + them as opposed to procuring a JDK ourselves. This commit replaces the + JDK in the Docker image with the bundled JDK. + +commit a478196f79a221d046f2c5c904cf14c5ce7fc8ff +Author: Martijn van Groningen +Date: Wed Mar 20 12:49:47 2019 +0100 + + Add realistic hlrc response serialization test base class (#39844) + + The base class facilitates generating a server side response test instance, + that gets serialized as xcontent, which then gets parsed into a hlrc response + instance, which then gets asserted against the server side response instance. + + This way of testing is more realistic then how hlrc response classes are tested + today, which basically tests that serialization works by generating + hlrc response instance, serialize that to xcontent and then parse it back + to a hlrc response instance. + + Besides adding a base test class, this change also cuts AcknowledgedResponseTests + and BroadcastResponseTests over to use this base class. + + Relates to #39745 + +commit 522f33e9eb593ae196c7f875ad791bac6637a5b4 +Author: David Turner +Date: Wed Mar 20 11:56:36 2019 +0000 + + Reword frozen indices introduction (#40160) + + Clarifies that we don't move anything extra to persistent storage when freezing + an index, and gives a bit more context for less experienced users. + +commit 235f57989fbb243a04d0a26914ce85bc27905404 +Author: Simon Willnauer +Date: Wed Mar 20 11:24:29 2019 +0100 + + Return cached segments stats if `include_unloaded_segments` is true (#39698) + + Today we don't return segments stats for closed indices which makes it + hard to tell how much memory such an index would require. With this change + we return the statistics if requested by setting `include_unloaded_segments` to + true on the rest request. + + Relates to #39512 + +commit 9ce740a2ebd2af87a28db802de8f0a78af3f7e2a +Author: Jason Tedor +Date: Wed Mar 20 00:05:30 2019 -0400 + + Modfiy casing in JVM home log message + + This makes the log message consistent with the following line that shows + the JVM arguments. + +commit c2a566a64e3cdcb86746653b1e89ff50bab8b83c +Author: Jason Tedor +Date: Tue Mar 19 23:44:19 2019 -0400 + + Upgrade bundled JDK and Docker images to JDK 12 (#40229) + + This commit bumps the JDK used the bundled JDK and the Docker images to + JDK 12. + +commit 1ec9fb3597b268e1045f883d14e2be3051bcbe31 +Author: Luca Cavanna +Date: Tue Mar 19 23:27:49 2019 +0100 + + Re-enable bwc tests (#40217) + + Relates to #40177 which is now merged and backported to all branches. + +commit 4f28d769cae1976ab0120d8feb1fad172855c534 +Author: Florian Kelbert +Date: Tue Mar 19 18:40:29 2019 +0100 + + Correction to ShingleTokenFilter in migration docs (#40208) + + Changed "Limit to the difference between max_shingle_size and min_shingle_size in ShingleTokenFilter" + +commit 696cb22e4a83ce32bc8682f280df55990fcb2c84 +Author: Lisa Cawley +Date: Tue Mar 19 11:00:42 2019 -0700 + + [DOCS] Enable testing for API key examples (#39583) + +commit 69f5869707bd63e9448e9664596a18677dcb0c17 +Author: Zachary Tong +Date: Tue Mar 19 13:30:43 2019 -0400 + + Mute SearchResponseMergerTests#testMergeSearchHits + + Tracking issue: https://github.com/elastic/elasticsearch/issues/40214 + +commit 33d8738c6859129b4af3a854c131f195366ceee4 +Author: David Turner +Date: Tue Mar 19 17:31:07 2019 +0000 + + Fix RareClusterStateIT on MacOS (#40203) + + Today RareClusterStateIT#testAssignmentWithJustAddedNodes fails on my Mac + because it waits for the default connection timeout of 30 seconds to connect to + a fake node with IP address 0.0.0.0. This connection attempt fails much more + quickly on Linux so the test passes. + + This commit fixes this by reducing the connection timeout for this test. + +commit a13b4bc8c55bcd36cfa94c1e2b60312cafbd945a +Author: Nhat Nguyen +Date: Tue Mar 19 09:13:27 2019 -0400 + + Always fail engine if delete operation fails (#40117) + + Unlike index operations which can fail at the document level to + analyzing errors, delete operations should never fail at the document + level whether soft-deletes is enabled or not. With this change, we will + always fail the engine if we fail to apply a delete operation to Lucene. + + Closes #33256 + +commit d58864745c6788dd2c2fa288e789d1d05abcf3e4 +Author: Nhat Nguyen +Date: Tue Mar 19 11:40:27 2019 -0400 + + Dump recovery if fail to get doc count with preference (#40168) + + With this change, we will dump the recovery state if we fail to get doc + count for a given index with a preference in rolling upgrade tests. We + should have more information to look into why the provided preference is + not valid. I also unmuted `testRelocationWithConcurrentIndexing` in this + change. + + Relates #34950 + +commit 85bb5a7f46705035454f8b81b309051035c160bc +Author: Gordon Brown +Date: Tue Mar 19 10:52:35 2019 -0600 + + Only count some fields types for deprecation check (#40166) + + Some field types are not used for queries which use auto-expansion, in + particular, `binary`, `geo_point`, and `geo_shape`. This was causing the + count returned by the deprecation check and the count returned by the + query-time deprecation warning to be misaligned for indices with fields + of those types, with the count returned by the deprecation check being + larger. + +commit a054a9866ecd81f99346211dc99ff0c0a4853f36 +Author: Mark Vovchuk <18199647+mark54g@users.noreply.github.com> +Date: Mon Mar 18 15:10:17 2019 -0400 + + Update managing-roles.asciidoc + +commit 4c9d7df88756d3ff9136f0c32a54dcd34bcf3326 +Author: Luca Cavanna +Date: Tue Mar 19 10:04:35 2019 +0100 + + Disable bwc tests + + Relates to #40177 + +commit d14e79e8493f57a638a00c0be8cec04abe8a7e55 +Author: Luca Cavanna +Date: Tue Mar 19 08:45:34 2019 +0100 + + Serialize top-level pipeline aggs as part of InternalAggregations (#40177) + + We currently convert pipeline aggregators to their corresponding + InternalAggregation instance as part of the final reduction phase. + They arrive to the coordinating node as part of QuerySearchResult + objects fom the shards and, despite we may incrementally reduce + aggs (hence we may have some non-final reduce and the final + one later) all the reduction phases happen on the same node. + + With CCS minimizing roundtrips though, each cluster performs its + own non-final reduction, and then serializes the results back to + the CCS coordinating node which will perform the final coordination. + This breaks the assumptions made up until now around reductions + happening all on the same node. + + With #40101 we have made sure that top-level pipeline aggs are not + reduced as part of the non-final reduction. The next step is to make + sure that they don't get lost, meaning that each coordinating node + needs to send them back to the CCS coordinating node as part of + the top-level `InternalAggregations` object. + + Closes #40059 + +commit 803ec46331b8db49b176a1cd2b1de68a42a0c41e +Author: Luca Cavanna +Date: Mon Mar 18 12:24:32 2019 +0100 + + Skip sibling pipeline aggregators reduction during non-final reduce (#40101) + + Today a coordinating node forces a final reduction of sibling pipeline aggregators whenever reducing aggs, unless it is reducing aggs incrementally. This works well for incremental reduction of aggs, but breaks CCS when minimizing roundtrips as each cluster ends up reducing its own pipeline aggregators locally while that should only be done by the CCS coordinating node later. This causes issues as after their reduction, pipeline aggs cannot be further reduced, which is what happens with CCS causing errors like "java.lang.UnsupportedOperationException: Not supported" being returned. + + Each coordinating node should rather honour the reduce context flag that + indicates whether we are executing a final reduce or not. If not, it should leave the sibling pipeline aggregations alone. + + Note that his bug affects only pipeline aggs that don't have a parent in + the aggs tree, while all the others work well. + + Relates to #40059 but does not fix it yet, as the CCS coordinating node also needs to be adapted to recreate sibling pipeline aggregators from the request. + +commit 83f12a3d9c86f2aea061c101db31416c813c6c02 +Author: Luca Cavanna +Date: Mon Mar 18 12:23:44 2019 +0100 + + CCS: skip empty search hits when minimizing round-trips (#40098) + + When minimizing round-trips, each cluster returns its own independent + search response. In case sort by field and/or field collapsing were + requested, when one cluster has no results to return, the information + about the field that sorting was based on (SortField array) as well as + the field (and the values) that collapsing was performed on are missing + in the search response. That causes problems as we can't build the + proper `TopDocs` instance which would need to be either `TopFieldDocs` + or `CollapseTopFieldDocs`. The merge routine expects that all the top + docs are of the same exact type which can't be guaranteed. Given that + the problematic results are empty, hence have no impact on the final + results, we can simply skip them. + + Relates to #32125 + Closes #40067 + +commit a11f1c88b37da84976704e1085cc9d5d2228f7a3 +Author: Luca Cavanna +Date: Mon Mar 18 11:37:56 2019 +0100 + + [DOCS] add details on version compatibility and remote gateway selection (#40056) + + This commit clarifies how the gateway selection works when configuring + remote clusters for CCR or CCS. Specifically, it clarifies compatibility + between different versions which is a very common question. + +commit 740020dfe8f08a96b762463d46b3dc89a3b40eca +Author: Alex Doerr +Date: Mon Mar 18 06:29:36 2019 -0400 + + Clarify version compatibility in snapshot/restore docs (#39329) + +commit 9c38fa6468b7dcb7e9ff5c70c22eb2e257f21f4b +Author: Luca Cavanna +Date: Thu Mar 14 16:16:33 2019 +0100 + + [TEST] Update TransportSearchActionTests#testShouldMinimizeRoundtrips + + Relates to #40044 + + Closes #40051 + +commit 07bfb4c7f73e7494698cc6f196226b4f358c70b1 +Author: Luca Cavanna +Date: Thu Mar 14 15:01:40 2019 +0100 + + CCS: Disable minimizing round-trips when dfs is requested (#40044) + + When using DFS_QUERY_THEN_FETCH search type, the dfs phase is run and + its results are used in the query phase to make scoring accurate. + When using CCS, depending on whether the DFS phase runs in the CCS + coordinating node (like if all shards were local) or in each remote + cluster (when minimizing round-trips), scoring will differ. + + This commit disables minimizing round-trips whenever DFS is requested, + as it is not currently possible to ensure that scoring is accurate in + that case. + + Relates to #32125 + +commit 387648065d06283dbf9e74356be5a979ab6b07d8 +Author: David Kyle +Date: Tue Mar 19 13:30:01 2019 +0000 + + [ML] Data Frame HLRC start & stop APIs (#40197) + +commit 8dc6862b1731a449d76ae3dfe6453cdf56e2cedf +Author: Nhat Nguyen +Date: Mon Mar 18 23:11:02 2019 -0400 + + Unmute and trace testPendingRefreshWithIntervalChange + + Tracked at #39565 + +commit dde41cc2dd39ff62c37dd0b27427fccedabf61ef +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Mar 19 11:50:21 2019 +0100 + + Node repurpose tool (#39403) + + When a node is repurposed to master/no-data or no-master/no-data, v7.x + will not start (see #37748 and #37347). The `elasticsearch repurpose` + tool can fix this by cleaning up the problematic data. + +commit c46dd6ad08bd200b76e8e98d11144b7a8bf1f550 +Author: Alexander Reelsen +Date: Tue Mar 19 09:56:20 2019 +0100 + + Replace java mail with jakarta mail (#40088) + + The eclipse foundation has taken over the javax mail dependency, which + resulted in a naming change of the dependency. + +commit 1d8b5fc658f2a298ae6b17d10a9741cd351ff5cb +Author: Yannick Welsch +Date: Tue Mar 19 08:54:41 2019 +0100 + + Fail command-line client's auto-URL detection with helpful message (#40151) + + The setup-passwords tool gives cryptic messages in case where custom discovery providers are + used (see #33580). As the URL auto-detection logic should be seen as best effort, this commit + improves the exception message to make it clearer what needs to be done to fix the issue. + + Relates #33580 + +commit cb29d3d8d0aac18ad323dbce25051d3ab6c980f2 +Author: Alpar Torok +Date: Tue Mar 19 08:02:11 2019 +0200 + + Fix build-tools unit test on windows (#39986) + +commit f88e4181ca21171e0157298d17ae341f5f059f81 +Author: Jason Tedor +Date: Mon Mar 18 21:47:52 2019 -0400 + + Enable reading auto-follow patterns from x-content (#40130) + + This named writable was never registered, so it means that we could not + read auto-follow patterns that were registered in the cluster + state. This causes them to be lost on restarts, a bad bug. This commit + addresses this by registering this named writable, and we add a basic + CCR restart test to ensure that CCR keeps functioning properly when the + follower is restarted. + +commit d093205b6a4ba96ec66f04717b888a1d072e50ff +Author: Lisa Cawley +Date: Mon Mar 18 14:12:23 2019 -0700 + + [DOCS] Fixes edit_url attributes that were externalized as text strings (#40161) + +commit afaa7e60f90c3ef4ca55f20f866601addc20e64a +Author: jaymode +Date: Mon Mar 18 15:02:37 2019 -0600 + + Mute FullClusterRestartIT#testWatcher + + This test has failed due to incorrect versions on the watches, see + +commit 95f660d577a0c40a279e6d2f5fd938cb9b9f9f58 +Author: Dimitris Athanasiou +Date: Tue Mar 5 12:54:50 2019 +0200 + + Mute NoMasterNodeIT.testNoMasterActionsWriteMasterBlock test (#39689) + + Relates #39688 + +commit 0b214c1bfbcf7301b948253be84e7c5ea9042f0a +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Mar 18 21:13:29 2019 +0100 + + Linearizability checker memory reduction (#40149) + + The cache used in linearizability checker now uses approximately 6x less + memory by changing the cache from a set of (bits, state) tuples into a + map from bits -> { state }. + + Each combination of states is kept once only, building on the + assumption that the number of state permutations is small compared to + the number of bits permutations. For those histories that are difficult + to check we will have many bits combinations that use the same state + permutations. + + We end up now using approximately 15 bytes per entry compared to 101 + bytes before, ie. a 6x improvement, allowing us to linearizability check + significantly longer histories. + + Re-enabled linearizability checker in CoordinatorTests, hoping above + ensures we no longer run out of memory. + + Resolves #39437 + +commit c8a4a7fc9df103f5cafb679e3c2c70bb78955d8b +Author: Gordon Brown +Date: Mon Mar 18 13:46:56 2019 -0600 + + Remove Migration Upgrade and Assistance APIs (#40075) + + The Migration Assistance API has been functionally replaced by the + Deprecation Info API, and the Migration Upgrade API is not used for the + transition from ES 6.x to 7.x, and does not need to be kept around to + repair indices that were not properly upgraded before upgrading the + cluster, as was the case in 6. + +commit 38e9522218cc391b8225e24fde950b6278942401 +Author: Nhat Nguyen +Date: Mon Mar 18 13:26:51 2019 -0400 + + Remove wait for cluster state step in peer recovery (#40004) + + We introduced WAIT_CLUSTERSTATE action in #19287 (5.0), but then stopped + using it since #25692 (6.0). This change removes that action and related + code in 7.x and 8.0. + + Relates #19287 + Relates #25692 + +commit 9ba0bdf5289709bba636ad6a16a8ffbbedab46aa +Author: Nhat Nguyen +Date: Sun Mar 17 11:23:35 2019 -0400 + + Dump cluster state if ensureGreen timed out in QA tests (#40133) + + When the method ensureGreen in QA tests is timed out, it does not + provide enough info for us to investigate why the testing index is + not green yet. With this change, we will dump the cluster state if + ensureGreen timed out. + + Relates #32027 + +commit d720a64b9e81352f8a09d743f24706737aa9f141 +Author: Nhat Nguyen +Date: Wed Mar 13 12:10:14 2019 -0400 + + Ensure sendBatch not called recursively (#39988) + + This PR introduces AsyncRecoveryTarget which executes remote calls of + peer recovery asynchronously. In this change, we also add a new + assertion to ensure that method sendBatch, which sends a batch of + history operations in phase2, is never called recursively on the same + thread. This new assertion will also be used in method sendFileChunks. + +commit 7325b2a3a786923f7a551058a77f998b4d8a88fb +Author: Lisa Cawley +Date: Mon Mar 18 11:11:52 2019 -0700 + + [DOCS] Moves monitoring settings (#40167) + +commit 42602478b8705e7f3ced7754a72e1f1779595670 +Author: Andrey Ershov +Date: Mon Mar 18 18:37:20 2019 +0100 + + Unmute, fix, refactor and zen2ify NetworkDisruptionIT (#38351) + + This commit unmutes NetworkDisruptionIT. + + It makes changes necessary for Zen2 - avoids usage of + autoMinMasterNodes and selects cluster size, such that there is no + need to call AddVotingExclusion. + This test also introduces refactors a single method + prepareDistruptedCluster to be used by both test methods. + Unfortunately, NetworkDisruption is broken and the + testNetworkPartitionRemovalRestoresConnections "is fixed" by + introducing assertBusy - #38348. + + Relates #36205 + Relates #38348 + + (cherry picked from commit 97707c7f892636e5b75c3df546b067414acb27cd) + +commit d35d9d1886d1e99ebe466205be993f792c90f684 +Author: Andrey Ershov +Date: Thu Mar 14 16:27:32 2019 +0100 + + cd ES_HOME in elasticsearch-env (#39937) + + This commit adds cd $ES_HOME to elasticsearch-env and removes it from + elasticsearch. This way, both elasticsearch and elasticsearch-cli are + executed with the working directory set to $ES_HOME. The need for the + fix arose from the following bug: + 1. Explicitly set path.data to relative to ES_HOME path in + elasticsearch.yml. + 2. Run elasticsearch from any directory. Elasticsearch is able to + correctly start. + 3. Stop elasticsearch. + 4. Run elasticsearch-node unsafe-bootstrap, not from ES_HOME directory. + It will fail with an exception. + This commit fixes the issue and adds a new test. + + This PR fixes the issue and adds a new test. + Also tests >=100 are renamed because alphabetic order does not work for + them. + + (cherry picked from commit 2ffc29306ff7366efc598e7b4dd2ce528895cd3a + with fixes by #40083 and #40118) + +commit 5be12e09995854138d827d0cc37f7e9a5989fd1c +Author: Jason Tedor +Date: Mon Mar 18 10:10:51 2019 -0400 + + Safe publication of AutoFollowCoordinator (#40153) + + We were leaking a reference to an AutoFollowCoordinator during + construction, violating safe publication according to the JLS + specification. This commit addresses this by waiting to register + AutoFollowCoordinator with the ClusterApplierService after the + AutoFollowCoordinator is fully constructed. We also remove ourselves as + a listener when stopping. + +commit 791814bb479c6757df34740da592cbb99f2f9b17 +Author: Andrei Stefan +Date: Mon Mar 18 15:35:35 2019 +0200 + + SQL: fix incorrect ordering of groupings (GROUP BY) based on orderings (ORDER BY) (#40087) + + * Take into consideration aliases that can be used as aggregates + and in the ORDER BY element so that the groupings are re-ordered inside + the composite aggregation according to the ORDER BY ordering. + + (cherry picked from commit 110c0b90b9cf2e9344ab3f412cfa8f8cd94ad71f) + +commit 076a68007c67f051040c33618aea131a1814fbd7 +Author: Costin Leau +Date: Mon Mar 18 14:56:00 2019 +0200 + + SQL: Add multi_value_field_leniency inside FieldHitExtractor (#40113) + + For cases where fields can have multi values, allow the behavior to be + customized through a dedicated configuration field. + By default this will be enabled on the drivers so that existing datasets + work instead of throwing an exception. + For regular SQL usage, the behavior is false so that the user is aware + of the underlying data. + + Fix #39700 + + (cherry picked from commit 2b351571961f172fd59290ee079126bbd081ceaf) + +commit b8ad337234ffe2f803137d564cd07d3d7ba684f9 +Author: Jason Tedor +Date: Mon Mar 18 07:24:51 2019 -0400 + + Stop auto-followers on shutdown (#40124) + + When shutting down a node, auto-followers will keep trying to run. This + is happening even as transport services and other components are being + closed. In some cases, this can lead to a stack overflow as we rapidly + try to check the license state of the remote cluster, can not because + the transport service is shutdown, and then immeidately retry + again. This can happen faster than the shutdown, and we die with stack + overflow. This commit adds a stop command to auto-followers so that this + retry loop occurs at most once on shutdown. + +commit eb540125ea6876b45500f65f7681605cbfaf29da +Author: Jim Ferenczi +Date: Mon Mar 18 11:33:54 2019 +0100 + + Fix IndexSearcherWrapper visibility (#39071) (#40145) + + This change adds a wrapper for IndexSearcher that makes IndexSearcher#search(List, Weight, Collector) visible by + sub-classes. The wrapper is used by the ContextIndexSearcher to call this protected method on a searcher created by a plugin. + This ensures that an override of the protected method in an IndexSearcherWrapper plugin is called when a search is executed. + + Closes #30758 + +commit 8a3f87bdcd193d8af68c8dd001aad45ecec90ee3 +Author: David Turner +Date: Mon Mar 18 09:12:03 2019 +0000 + + Note that GET /_cluster/state is unstable (#40104) + + The `GET /_cluster/state` API returns an internal representation of the cluster + state that does change from version to version. It's useful for debugging, but + it is not intended for regular use by clients. + + This change adjusts the documentation of `GET /_cluster/state` to clarify that + this API yields an internal representation that should not be expected to + remain stable between versions. + + Relates #40061, #40016 + +commit 5b73a1bc7dd2e6079ddaf89f238fdbb8de8a054e +Author: Jim Ferenczi +Date: Mon Mar 18 09:32:45 2019 +0100 + + Add an option to force the numeric type of a field sort (#38095) (#40084) + + This change adds an option to the `FieldSortBuilder` that allows to transform the type + of a numeric field into another. Possible values for this option are `long` that transforms + the source field into an integer and `double` that transforms the source field into a floating point. + This new option is useful for cross-index search when the sort field is mapped differently on some + indices. For instance if a field is mapped as a floating point in one index and as an integer in another + it is possible to align the type for both indices using the `numeric_type` option: + + ``` + { + "sort": { + "field": "my_field", + "numeric_type": "double" <1> + } + } + ``` + + <1> Ensure that values for this field are transformed to a floating point if needed. + +commit 95024798c089b68703c244e15e39faa470b9fbff +Author: Daniel Mitterdorfer +Date: Mon Mar 18 08:59:02 2019 +0100 + + Document monitoring node stats collection timeout (#39846) (#40140) + + With this commit we document the setting + `xpack.monitoring.collection.node.stats.timeout` that has been missing + so far in the docs. + + Supersedes #31043 + +commit 607d05f0b814156c2e8f52c93c7af24d6dd33792 +Author: Ioannis Kakavas +Date: Mon Mar 18 08:52:14 2019 +0200 + + Enable QA tests to run with FIPS nodes (#40105) + + This commit enables full-cluster-restart and rolling-upgrade tests + to run with nodes using a JVM in fips approved only node by using + PEM key material instead of a JKS for the transport layer in that + case. + +commit 3b9a884f92960f81dbf2aa47631c430908ed8910 +Author: Ioannis Kakavas +Date: Mon Mar 18 08:45:50 2019 +0200 + + Throw an exception when unable to read Certificate (#40092) + + With SUN security provider, a CertificateException is thrown when + attempting to parse a Certificate from a PEM file on disk with + `sun.security.provider.X509Provider#parseX509orPKCS7Cert` + + When using the BouncyCastle Security provider (as we do in fips + tests) the parsing happens in + CertificateFactory#engineGenerateCertificates which doesn't throw + an exception but returns an empty list. + + In order to have a consistent behavior, this change makes it so + that we throw a CertificateException when attempting to read + a PEM file from disk and failing to do so in either Security + Provider + + Resolves: #39580 + +commit 124de8d9386f0ad5f6dcdae6c38ff91692cd2e07 +Author: Albert Zaharovits +Date: Sun Mar 17 14:45:08 2019 +0200 + + Un-hardcode SecurityIndexManager to handle generic indices (#40064) + + `SecurityIndexManager` is hardcoded to handle only the `.security`-`.security-7` alias-index pair. + This commit removes the hardcoded bits, so that the `SecurityIndexManager` can be reused + for other indices, such as the planned security tokens index (`.security-tokens-7`). + +commit 1b75ee0bd70eeb6eb69a344324abbc3bbe1a2e58 +Author: Albert Zaharovits +Date: Sun Mar 17 13:03:50 2019 +0200 + + AuditTrail correctly handle ReplicatedWriteRequest (#39925) + + This fix deduplicates index names in `BulkShardRequests` and only audits + the specific resolved index for every comprising `BulkItemRequest`. + +commit a77e3d1ad851bd303a4f7b48116ca50647d8d6ff +Author: Ioannis Kakavas +Date: Sun Mar 17 11:45:58 2019 +0200 + + Adjust ldap timeout for idp fixture (#40102) + + This change adjusts the LDAP connection timeout for retrieving + attributes while performing the SAML IT to 5 seconds, from 5 ms + that it previously was. + Resolves: #40025 + +commit 64028f3d8fee6b581a08a78eda3244176369c819 +Author: David Roberts +Date: Sun Mar 17 07:48:42 2019 +0000 + + Mute JobResultsProviderIT.testMultipleSimultaneousJobCreations + + Due to https://github.com/elastic/elasticsearch/issues/40134 + +commit 2df0405d9732fe23785fb1242b45a607990cde5d +Author: Jason Tedor +Date: Sat Mar 16 14:39:55 2019 -0400 + + Reenable BWC tests after removing cluster state size (#40127) + + This commit reenables the BWC tests after removing cluster state size + and backporting that work. + +commit 28729eb54c1d125e067e97da081ffc54f9c9edbc +Author: Benjamin Trent +Date: Sat Mar 16 17:14:07 2019 -0500 + + [ML] fixing sort order (#40119) (#40123) + +commit 0824eceacffcda2c5d677b81c490f1f3206b230b +Author: Jason Tedor +Date: Sat Mar 16 10:36:02 2019 -0400 + + Add log message for auto-follower timeout + + When an auto-follower coordinator times out waiting for the remote + cluster state, we do not log any indication of this. While this is + expected behavior in quiet deployments, it is still useful to see this + information for tracing the behavior of the auto-follow + coordinator. This commit adds a trace log message indicating that the + timeout. + +commit 86d1d03c37768ca24cc4b5a2a702db6a828a3222 +Author: Jason Tedor +Date: Fri Mar 15 17:16:25 2019 -0400 + + Remove cluster state size (#40109) + + This commit removes the cluster state size field from the cluster state + response, and drops the backwards compatibility layer added in 6.7.0 to + continue to support this field. As calculation of this field was + expensive and had dubious value, we have elected to remove this field. + +commit a019af7690f08c8c3512325b1d3a862eb3919e93 +Author: Igor Motov +Date: Fri Mar 15 13:15:11 2019 -0400 + + SQL: Refactor Literals serialization method (#40058) + + Since other classes besides intervals can be serialized as part of + the Cursor, the getNamedWritables method should be moved from Intervals + to a more generic class Literals. + + Relates to #39973 + +commit 0b50a670a413575fc6bf6661fc046becee354993 +Author: Tim Brooks +Date: Fri Mar 15 11:47:29 2019 -0600 + + Remove transport name from tcp channel (#40074) + + Currently, we maintain a transport name ("mock-nio", "nio", "netty") + that is passed to a `TcpTransportChannel` when a request is received. + The value of this name is to associate with the task when we register a + task with the task manager. However, it is only possible to run ES with + one transport, so having an implementation specific name is unnecessary. + This commit removes the name and replaces it with the generic + "transport". + +commit 6ffa8a040dad6962353f0395bbed3538e1fd0a45 +Author: Julie Tibshirani +Date: Fri Mar 15 10:34:58 2019 -0700 + + Document the limitation around field aliases and percolator. (#40073) + + Currently if a field alias is updated, any percolator queries that contain the + alias will still refer to its old target. This PR documents the issue while we + look into addressing it. + + Relates to #37212. + +commit c72feedd749f8007779ff45c2e010be47f85403b +Author: Zachary Tong +Date: Fri Mar 15 11:46:00 2019 -0400 + + Do not allow Sampler to allocate more than maxDoc size, better CB accounting (#39381) + + The `sampler` agg creates a BestDocsDeferringCollector, which internally + initializes a priority queue of size `shardSize`. This queue is + populated with empty `Object` sentinels, which is roughly 16b per + object. + + Similarly, the Diversified samplers create a DiversifiedTopDocsCollectors + which internally track PQ slots with ScoreDocKeys, weighing in around + 28kb + + If the user sets a very abusive `shard_size`, this could easily OOM + a node or cluster since these PQ are allocated up-front without + any checks. + + This commit makes sure that when we create the collector, it + cannot be larger than the maxDoc so that we don't accidentally blow + up the node. We ensure the size is not greater than the overall + index maxDoc. A similar treatment is done for `maxDocsPerValue` + parameter of the diversified samplers + + For good measure, this also adds in some CB accounting to try and track + memory usage. + + Finally, a redundant array creation is removed to reduce a bit of + temporary memory. + +commit 4d2ceb5ed50180bd79e1771409684a603932f322 +Author: MK Swanson +Date: Fri Mar 15 13:17:34 2019 -0400 + + [DOCS] Edited intro sentence for clarity. Closes #39865 (#40108) + +commit dcaabdfce890ea5dc2a91eccb231ddf3d3a15c95 +Author: Jack Conradson +Date: Fri Mar 15 09:33:08 2019 -0700 + + Add Painless cast tests for long and Long (#40007) + +commit c74111ff8ec9ac2c12564c60425818b765a49c0e +Author: Yannick Welsch +Date: Fri Mar 15 17:14:00 2019 +0100 + + Reduce logging noise when stepping down as master before state recovery (#39950) + + Reduces the logging noise from the state recovery component when there are duelling elections. + + Relates to #32006 + +commit 4eb3683d65265bd30881468cdf98232c2a48c230 +Author: David Kyle +Date: Fri Mar 15 13:17:08 2019 +0000 + + Mute CcrRetentionLeaseIT tests (#40090) + +commit c92476f591668b3f4a43fbdf3fb3df5bbafc5357 +Author: Lisa Cawley +Date: Fri Mar 15 07:54:45 2019 -0700 + + [DOCS] Replaces CCS terms with attributes (#40076) + +commit 09809bc91b8a2e561171978d081d6094416ac0e2 +Author: David Kyle +Date: Fri Mar 15 12:16:38 2019 +0000 + + [ML] Avoid assertions on empty Optional in DF usage test (#40043) + + Refactor the usage class to make testing simpler + +commit 95e61d4bb187fd1d66245a99058b0b6f95b0939b +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Mar 15 12:31:03 2019 +0100 + + Blob Store compress default false (#40054) + + Fixed documentation to comply with code (compress=false + is default until 8.0). + +commit 8d01b119189e06f8abc4b5bd2a3b6c7532494fae +Author: David Roberts +Date: Fri Mar 15 09:25:55 2019 +0000 + + [ML] Fix race condition when creating multiple jobs (#40049) + + If multiple jobs are created together and the anomaly + results index does not exist then some of the jobs could + fail to update the mappings of the results index. This + lead them to fail to write their results correctly later. + + Although this scenario sounds rare, it is exactly what + happens if the user creates their first jobs using the + Nginx module in the ML UI. + + This change fixes the problem by updating the mappings + of the results index if it is found to exist during a + creation attempt. + + Fixes #38785 + +commit 78a97543184262b522a014a12281db38d9e273f2 +Author: David Kyle +Date: Fri Mar 15 09:06:49 2019 +0000 + + Mute test NetworkDisruptionIT.testJobRelocation + Relates to #39858 + +commit 3960374a6ff45546b8047d33a23edc550fe85db8 +Author: Costin Leau +Date: Fri Mar 15 11:45:10 2019 +0200 + + SQL: Introduce MAD (MedianAbsoluteDeviation) aggregation (#40048) + + Add Median Absolute Deviation aggregation + + Fix #39597 + + (cherry picked from commit 4f09613942a9249d06c74da64ad7e6f362e97f56) + +commit 0d152a54f8b9907e11120ba88356d003f980de5f +Author: David Turner +Date: Fri Mar 15 08:04:08 2019 +0000 + + Await all pending activity in testConnectAndDisconnect (#40037) + + We call `ensureConnections()` to undo the effects of a disruption. However, it + is possible that one or more targets are currently CONNECTING and have been + since the disruption was active, and that the connection attempt was thwarted + by a concurrent disruption to the connection. If so, we cannot simply add our + listener to the queue because it will be notified when this CONNECTING activity + completes even though it was disrupted. We must therefore wait for all the + current activity to finish and then go through and reconnect to any missing + nodes. + + Closes #40030. + +commit 8f09c77777024500be62330328053ade640cfb2c +Author: Ryan Ernst +Date: Fri Mar 15 00:55:15 2019 -0700 + + Add no-jdk distributions (#39882) + + This commit adds a variant for every official distribution that omits + the bundled jdk. The "no-jdk" naming is conveyed through the package + classifier, alongside the platform. Package tests are also added for + each new distribution. + +commit a323132503dbeb0b98f592b66caf12f1dcb45f38 +Author: David Turner +Date: Fri Mar 15 07:36:05 2019 +0000 + + Create retention leases file during recovery (#39359) + + Today we load the shard history retention leases from disk whenever opening the + engine, and treat a missing file as an empty set of leases. However in some + cases this is inappropriate: we might be restoring from a snapshot (if the + target index already exists then there may be leases on disk) or + force-allocating a stale primary, and in neither case does it make sense to + restore the retention leases from disk. + + With this change we write an empty retention leases file during recovery, + except for the following cases: + + - During peer recovery the on-disk leases may be accurate and could be needed + if the recovery target is made into a primary. + + - During recovery from an existing store, as long as we are not + force-allocating a stale primary. + + Relates #37165 + +commit 8d2184b3151d6a14ee51b77bd9eeb5076f999ec5 +Author: David Turner +Date: Fri Mar 15 07:44:31 2019 +0000 + + Fix up committed configuration on fake Zen1 nodes (#40065) + + Today we test Zen1/Zen2 compatibility by running 7.x nodes with a "fake" Zen1 + implementation. However this is not a truly faithful test because these nodes + do known how to properly deserialize a 7.x cluster state, voting configurations + and all, whereas a real Zen1 node is in 6.7 and ignores the coordination + metadata. + + We only ever apply a cluster state that's been committed, which in Zen2 + involves setting the last-committed configuration to equal the last-accepted + configuration. Zen1 knows nothing about this adjustment, so it is possible for + these to differ. This breaks the assertion that the cluster states are equal on + all nodes after integration tests. + + This commit fixes this by implementing this adjustment in Zen1 before applying + a cluster state. + + Fixes #40055. + +commit 35aaf04c8c7013dd21500ea6d90406f5fba606be +Author: Ioannis Kakavas +Date: Fri Mar 15 09:36:51 2019 +0200 + + Handle empty input in AddStringKeyStoreCommand (#39490) + + This change ensures that we do not make assumptions about the length + of the input that we can read from the stdin. It still consumes only + one line, as the previous implementation + +commit e050d507598eb248f9067c635efbe9b6b42476ba +Author: Lisa Cawley +Date: Thu Mar 14 14:22:06 2019 -0700 + + [DOCS] Removes X-Pack settings section (#39870) + +commit b57af6c40187913ac65be5a96a4a6dbbe4d45c44 +Author: Jack Conradson +Date: Thu Mar 14 09:18:57 2019 -0700 + + Add a Painless Context REST API (#39382) + + This PR adds an internal REST API for querying context information about + Painless whitelists. + + Commands include the following: + GET /_scripts/painless/_context -- retrieves a list of contexts + GET /_scripts/painless/_context?context=%name% retrieves all available + information about the API for this specific context + +commit ae0ff05c900be0063fb5178da71a88065b8a1399 +Author: Ioannis Kakavas +Date: Thu Mar 14 20:23:54 2019 +0200 + + Adjust testGetSslCertificates to run in FIPS (#40046) + + As discovered in #40041, when parsing certificates from files, the + SUN Security Provider normalizes DNs from parsed certificates by + adding spaces between RDNs, while the BouncyCastle one (which we + use in FIPS tests) does not. + + We could proceed to normalize the DNs in the same manner in this + test by using i.e. the Unbound LDAP SDK but since the goal of this + test is to validate that we do get to read these exact certificates + from our trust sources and not to validate subject DNs, this commit + changes the test to check the serial number instead + + Resolves: #40041 + +commit e2b60c71412e73d7b839b9f145782fafe5865690 +Author: Tamara Braun +Date: Thu Mar 14 18:40:01 2019 +0100 + + Fix not Recognizing Disabled Object Mapper (#39862) + + * Fixes not finding disabled object mapper when using dotted field name + notation + * Closes #39456 + +commit 52b0b9cb55ad5b62cfa4ba47d1283e797dc7f726 +Author: Lisa Cawley +Date: Thu Mar 14 10:26:22 2019 -0700 + + [DOCS] Add reindex steps for internal indices (#40019) + +commit 8dc8fc507d95d86f4ee707e6b5ffda15d5415801 +Author: Ioannis Kakavas +Date: Thu Mar 14 18:02:28 2019 +0200 + + Handle UTF-8 values in the keystore (#39496) + + * Handle UTF8 values in the keystore + + Our current implementation uses CharBuffer#array to get the chars + that were decoded from the UTF-8 bytes. The backing array of + CharBuffer is created in CharsetDecoder#decode and gets an initial + length that is the same as the length of the ByteBuffer it decodes, + hence the number of UTF-8 bytes. + This works fine for the first 128 characters where each one needs + one bytes, but for the next UTF-8 characters (other latin alphabets + Greek, Cyrillic etc.) where we need 2 to 4 bytes per character, this + backing char array has a larger size than the number of the actual + chars this CharBuffer contains. Calling `array()` on it will return + a char array that can potentially have extra null chars so the + SecureString we get from the KeystoreWrapper, is not the same as the + one we entered. + + This commit changes the behavior to use Arrays#copyOfRange to get + the necessary chars from the CharBuffer and adds a test with + random ( maybe not printable ) UTF-8 strings + +commit e9fa7767ecb7a4b2d3084e72a31723b314c3a9c7 +Author: Jake Landis +Date: Wed Mar 13 14:22:21 2019 -0500 + + Fix test which still uses default type (#39997) + + org.elasticsearch.xpack.monitoring.action.MonitoringBulkRequestTests#testAddRequestContent + can still randomly use a defaultType for monitoring. The defaultType + support has been removed as of PR #39888. Prior to its's removal it + would default the type if one is not specified. The _type on the monitoring + bulk end point is currently required, though it is not used as the final index type + (which defaultType would have). + + Closes #39980 + +commit c02f49e9d305ca504532be19bc803e83cc3f10dc +Author: David Kyle +Date: Thu Mar 14 14:57:12 2019 +0000 + + [ML-Dataframe] Add Data Frame client to the Java HLRC (#40040) + + Adds DataFrameClient to the Java HLRC and implements PUT and + DELETE data frame transform. + +commit 24973cf464088f80f72d56c9ddacd8c32e3f90e7 +Author: Jason Tedor +Date: Thu Mar 14 09:39:39 2019 -0400 + + Adjust BWC version on cluster state size response + + This work has been backported all the way now, so this commit adjusts + the BWC version. + + Relates #40016 + +commit 9181668edf7186ad15d06d8fc528c7a1408b9883 +Author: Jason Tedor +Date: Thu Mar 14 08:56:19 2019 -0400 + + Stop returning cluster state size by default (#40016) + + Computing the compressed size of the cluster state on every invocation + of cluster:monitor/state action is expensive, and the value of this + field is dubious anyway. Therefore we want to remove computing this + field. As a first step, we stop computing and return this field by + default. To avoid breaking users, we will give them a system property to + use to tide them over until the next major release when we will actually + remove this field. This comes with a deprecation warning too, and the + backport to the appropriate minor will also include a note in the + migration guide. There will be a follow-up to remove this field in the + next major version. + +commit d02bca13147973ff751713f60c0b6e7348f5951d +Author: Jason Tedor +Date: Thu Mar 14 08:54:14 2019 -0400 + + Upgrade the bouncycastle dependency to 1.61 (#40017) + + This commit upgrades the bouncycastle dependency from 1.59 to 1.61. + +commit 4e9657f93fbda0d4d1560024019e5128dd53447a +Author: Marios Trivyzas +Date: Thu Mar 14 12:46:08 2019 +0100 + + SQL: Fix bug with JDBC timezone setting and DATE type (#39978) + + Previously, JDBC's REST call to the server was always sending UTC + instead of the timezone passed through connection string/properties. + + Moreover the conversion to java.sql.Date was problematic as a + calculation on the epoch millis was used to set the time to 00:00:00.000 + and the timezone info was lost. This caused the resulting java.sql.Date + object which is always using the JVM's timezone (no matter what timezone + setting is used in the connection string/properties) to be wrongly created. + + Fixes: #39915 + +commit 2512cf3ec8bb46849bdc93fcc4a3d2710c812000 +Author: Ioannis Kakavas +Date: Thu Mar 14 14:21:11 2019 +0200 + + Mute testGetSslCertificates in FIPS (#40042) + +commit 59201915db3e14c2163983fdce8d7d5501a8867c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Mar 14 13:36:11 2019 +1100 + + Mute DataFrameFeatureSetTests#testUsage test (#40023) + +commit 23619477310b98decb443879074d309296da90df +Author: Ioannis Kakavas +Date: Thu Mar 14 09:51:57 2019 +0200 + + Log response entity when submitting IDP form + + This will allow to troubleshoot the intermittent errors in + SamlAuthneticationIT + + Relates to https://github.com/elastic/elasticsearch/issues/40025 + +commit 4d1305b6dfff3fc4222083b6fb0ae0ef0bafabda +Author: Andrei Stefan +Date: Thu Mar 14 11:19:21 2019 +0200 + + SQL: Extend the multi dot field notation extraction to lists of values (#39823) + + (cherry picked from commit 300ae485dd08373727ca111a4d21276dd47d9a27) + +commit 20e5994179b899fd4b18086963ca0e25d8200a0c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Mar 14 19:40:15 2019 +1100 + + Mute failing tests in NodeConnectionsServiceTests (#40034) (#40035) + +commit 2016e2328546e95b1b3c25af1084343dde1b0f57 +Author: Benjamin Trent +Date: Wed Mar 13 17:08:43 2019 -0500 + + [ML] Refactor common utils out of ML plugin to XPack.Core (#39976) (#40009) + + * [ML] Refactor common utils out of ML plugin to XPack.Core + + * implementing GET filters with abstract transport + + * removing added rest param + + * adjusting how defaults can be supplied + +commit 8c6ff5de31b73f667f7e371fe560fa94a1952b3a +Author: Benjamin Trent +Date: Wed Mar 13 17:08:15 2019 -0500 + + [Data Frame] Refactor PUT transform to not create a task (#39934) (#40010) + + * [Data Frame] Refactor PUT transform such that: + + * POST _start creates the task and starts it + * GET transforms queries docs instead of tasks + * POST _stop verifies the stored config exists before trying to stop + the task + + * Addressing PR comments + + * Refactoring DataFrameFeatureSet#usage, decreasing size returned getTransformConfigurations + + * fixing failing usage test + +commit a3ef565a250e6353bfdc9235fbbe9e6ee99d4148 +Author: debadair +Date: Wed Mar 13 14:38:13 2019 -0700 + + [DOCS] First pass at upgrade updates for 7.0. (#39944) + + * [DOCS] First pass at upgrade updates for 7.0. + + * [DOCS] Updates X-Pack terminology + + * [DOCS] Incorporated feedback from lcawl. + +commit 4a42e408c5034740c7ee7586a81ee7b853b84a84 +Author: Igor Motov +Date: Wed Mar 13 15:35:18 2019 -0400 + + GEO: Add support for z values to libs/geo classes (#38921) + + Adds support for z-values to all Geometry objects in the + libs/geo library. + +commit 8a314a36db55381edb77d28602533263255fd72e +Author: Przemyslaw Gomulka +Date: Wed Mar 13 18:27:37 2019 +0100 + + Change zone formatting for all printers backport(#39568) #39952 + + After the joda-java time migration we were formatting zone ids with zoneOrOffsetId method. This when a date was provided with a ZoneRegion for instance America/Edmonton it was appending this zone identifier instead of zone formatted as +HH:MM. + This fix is changing the format of zone suffix for all printers and also always wrapping a Temporal into a ZonedDateTime when formatting. + + closes #38471 + backport #39568 + +commit 9b8b47114e5afe3b15d64e5052624633ce61dff4 +Author: Alpar Torok +Date: Wed Mar 13 19:11:50 2019 +0200 + + Configure TMP for test nodes on Windows (#39959) + + This breaks on windows where TMP dir default to C:\Windows and startup + fails with a permission error. + I tried to create a tmp dir and pass in `TMP` env, but it lead to a + class not found error, and since testclusers is already independent of + the calling environment I stopped there. + +commit 7a7658707a690ad0f1f70e7bf52ecbd8af6cc264 +Author: Jim Ferenczi +Date: Wed Mar 13 18:11:50 2019 +0100 + + Upgrade to Lucene release 8.0.0 (#39998) + + This commit upgrades to the GA release of Lucene 8 + + Closes #39640 + +commit 352f9f1f39fde994a31377fe31f03b6e4be8e07d +Author: Tim Brooks +Date: Wed Mar 13 09:01:01 2019 -0600 + + Remove sizing from `Recycler#obtain` (#39975) + + Currently there is a method `Recycler#obtain(size)` that allows a size + parameter to be passed. However all implementations ignore this + parameter and just allocate a page size based on other settings. This + commit removes this method. + +commit 87ec5116849bbbcc345e5079dc08d0a61463eadd +Author: Ioannis Kakavas +Date: Wed Mar 13 17:05:34 2019 +0200 + + Mute locale dependent mapping tests (#39996) + +commit 9731ba4338f7fdd2e883cc721df06130aa250d6d +Author: Adrien Grand +Date: Wed Mar 13 15:04:41 2019 +0100 + + Make the `type` parameter optional when percolating existing documents. (#39987) (#39989) + + `document_type` is the type to use for parsing the document to percolate, which + is already optional and deprecated. However `percotale` queries also have the + ability to percolate existing documents, identified by an index, an id and a + type. This change makes the latter optional and deprecated. + + Closes #39963 + +commit 16e4499b977b87f2116d1cdc4ea00365b21cf8cf +Author: Jason Tedor +Date: Wed Mar 13 10:02:17 2019 -0400 + + Fix CCR forget follower docs example + + This example was missing sample values in the forget follower API + call. This commit addresses this. + +commit 9300826d8a81ede8161a6adf948fbc68c2a40641 +Author: Andrey Ershov +Date: Wed Mar 13 15:06:25 2019 +0100 + + Do not log unsuccessful join attempt each time (#39756) + + When performing the test with 57 master-eligible nodes and one node + crash, we saw messy elections, when multiple nodes were attempting to + become master. + JoinHelper has logged 105 long log messages with lengthy stack + traces during one such election. + To address this, we decided to log these messages every time only on + debug level. + We will log last unsuccessful join attempt (along with a timestamp) + if any with WARN level if the cluster is failing to form. + + (cherry picked from commit 17a148cc27b5ac6c2e04ef5ae344da05a8a90902) + +commit 8f7ab84c5cc9ff7dda46aa4794f6ea7b93576934 +Author: Martijn van Groningen +Date: Wed Mar 13 10:10:27 2019 +0100 + + Add minimal docs around upgrading clusters with ccr enabled (#38037) + +commit 79e414df8672939b8549e435c6e19217615249b3 +Author: Dimitris Athanasiou +Date: Wed Mar 13 09:09:07 2019 +0200 + + [ML] Fix datafeed skipping first bucket after lookback when aggs are … (#39859) (#39958) + + The problem here was that `DatafeedJob` was updating the last end time searched + based on the `now` even though when there are aggregations, the extactor will + only search up to the floor of `now` against the histogram interval. + This commit fixes the issue by using the end time as calculated by the extractor. + + It also adds an integration test that uses aggregations. This test would fail + before this fix. Unfortunately the test is slow as we need to wait for the + datafeed to work in real time. + + Closes #39842 + +commit b8733eab000f6863a8e41665dc672eb9d597ca97 +Author: Ioannis Kakavas +Date: Tue Mar 12 23:54:23 2019 +0200 + + Replace Vagrant with Docker for idp-fixture (#39948) + + The change replaces the Vagrant box based fixture with a fixture + based on docker compose and 2 docker images, one for an openldap + server and one for a Shibboleth SAML Identity Provider. + + The configuration of both openldap and shibboleth is identical to + the previous one, in order to minimize required changes in the + tests + +commit b10dd3769c701acc982a04ed26e8677754a80112 +Author: Christoph Büscher +Date: Tue Mar 12 23:46:55 2019 +0100 + + Add analysis modes to restrict token filter use contexts (#36103) + + Currently token filter settings are treated as fixed once they are declared and + used in an analyzer. This is done to prevent changes in analyzers that are already + used actively to index documents, since changes to the analysis chain could + corrupt the index. However, it would be safe to allow updates to token + filters at search time ("search_analyzer"). This change introduces a new + property of token filters that allows to mark them as only being usable at search + or at index time. Any analyzer that uses these tokenfilters inherits that property + and can be rejected if they are used in other contexts. This is a first step towards + making specific token filters (e.g. synonym filter) updateable. + + Relates to #29051 + +commit db04288d148c73e3a4793f335adc234a803d0b03 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Mar 13 09:23:32 2019 +1100 + + Add pre-upgrade check to test cluster routing allocation is enabled (#39340) (#39815) + + When following the steps mentioned in upgrade guide + https://www.elastic.co/guide/en/elastic-stack/6.6/upgrading-elastic-stack.html + if we disable the cluster shard allocation but fail to enable it after + upgrading the nodes and plugins, the next step of upgrading internal + indices fails. As we did not check the bulk request response for reindexing, + we delete the old index assuming it has been created. This is fatal + as we cannot recover from this state. + + This commit adds a pre-upgrade check to test the cluster shard + allocation setting and fail upgrade if it is disabled. In case there + are search or bulk failures then we remove the read-only block and + fail the upgrade index request. + + Closes #39339 + +commit 26983d1fdf90a7527f2e7a42608e18e05d517226 +Author: Lisa Cawley +Date: Tue Mar 12 14:27:17 2019 -0700 + + [DOCS] Replaces CCR terms with attributes (#39516) + +commit efd7003ea9eec1106cfdd8fb807fd809d08812b5 +Author: Lisa Cawley +Date: Tue Mar 12 13:25:16 2019 -0700 + + [DOCS] Corrected API path for invalidate token and SSL certificate examples (#39530) + +commit e2b88bc706ccf0ae38ea47f85c83b76d3a6b34aa +Author: Andy Bristol +Date: Tue Mar 12 12:58:00 2019 -0700 + + add version 6.6.3 + +commit 77e46b4f53ef833095e281ec75905d4f3b502943 +Author: Lisa Cawley +Date: Tue Mar 12 13:02:00 2019 -0700 + + [DOCS] Added upgrade step for realm settings (#39672) + +commit 049970af3ef4de127601474ce9db33c590e40e54 +Author: David Turner +Date: Tue Mar 12 19:26:25 2019 +0000 + + Only connect to new nodes on new cluster state (#39629) + + Today, when applying new cluster state we attempt to connect to all of its + nodes as a blocking part of the application process. This is the right thing to + do with new nodes, and is a no-op on any already-connected nodes, but is + questionable on known nodes from which we are currently disconnected: there is + a risk that we are partitioned from these nodes so that any attempt to connect + to them will hang until it times out. This can dramatically slow down the + application of new cluster states which hinders the recovery of the cluster + during certain kinds of partition. + + If nodes are disconnected from the master then it is likely that they are to be + removed as part of a subsequent cluster state update, so there's no need to try + and reconnect to them like this. Moreover there is no need to attempt to + reconnect to disconnected nodes as part of the cluster state application + process, because we periodically try and reconnect to any disconnected nodes, + and handle their disconnectedness reasonably gracefully in the meantime. + + This commit alters this behaviour to avoid reconnecting to known nodes during + cluster state application. + + Resolves #29025. + +commit aeb0116355536dfb63ef1e3d8cee6b4d178bd35b +Author: Jack Conradson +Date: Tue Mar 12 10:12:42 2019 -0700 + + Add Painless cast tests for int and Integer (#39813) + +commit 9ab2410436d6b7e286ae7d727681b46d9f2fe9ea +Author: Tal Levy +Date: Tue Mar 12 11:14:41 2019 -0700 + + Adding an example in the Set processor documentation to address #30604 (#39941) (#39969) + + * Added an example of using set to copy values from one field to another. + + * Modified the document type to match the test. + +commit 8c78fc096d85c2de36a5eec83e58e2362340d949 +Author: Michael Basnight +Date: Tue Mar 12 11:00:02 2019 -0500 + + More lenient socket binding in LDAP tests (#39864) + + The LDAP tests attempt to bind all interfaces, + but if for some reason an interface can't be bound + the tests will stall until the suite times out. + + This modifies the tests to be a bit more lenient and allow + some binding to fail so long as at least one succeeds. + This allows the test to continue even in more antagonistic + environments. + +commit ba40230c7fb88be449495f5777206fb0e385cf1a +Author: Jay Modi +Date: Tue Mar 12 09:03:41 2019 -0600 + + Add client jar for transport-nio (#39860) + + This change marks the transport-nio plugin as having a client jar. The + nio transport can be used from a transport client and the + x-pack-transport artifact depends on the transport-nio jar but this jar + is not published. This change marks the transport-nio project as having + a client jar so that the jar may be published in the same way that we + publish the netty4 transport artifact. + + The change to actually publish the jar will be handled separately as an + update to the release manager. + +commit da67c2f7f8b04809dd642918383cf3bee82d9b72 +Author: Gordon Brown +Date: Tue Mar 12 09:06:31 2019 -0600 + + Deprecation check for indices with very large numbers of fields (#39869) + + Indices with very large numbers of fields (>1024 by default) that do not + have index.query.default_field set will experience query failures in 7.0 + for Simple Query String and Multi-Match queries. This deprecation check + issues a warning for indices of that size that do not have + index.query.default_field set. + + This also adds a deprecation check for index templates with field counts + that would trigger these query failures as well. + +commit aa967edd73d3881f6209d7d6c716a2e28d407cc3 +Author: Andrey Ershov +Date: Tue Mar 12 14:43:29 2019 +0100 + + Add elasticsearch-node tool docs (#37812) + + This commit, mostly authored by @DaveCTurner, + adds documentation for elasticsearch-node tool #37696. + + (cherry picked from commit 09425d5a5158c2d3fdad794411b3bbc4bba47b15) + +commit a29bba4ede7ee6b9592e2723eccbf0db0f23964d +Author: Przemyslaw Gomulka +Date: Tue Mar 12 12:10:36 2019 +0100 + + Migrate Streamable to writeable for index package backport(#37381) #39949 + + Migrate streamable classes from index package to Writeable and clean up access modifiers + + Related to #34389 + backport#37381 + +commit ad55e5b80d3d538747c3e75bda2ca4a80bd398c2 +Author: lzh3636 +Date: Tue Mar 12 05:41:15 2019 -0400 + + Log missing file exception when failing to read metadata snapshot (#32920) + + Adds the exception to the logged output, which contains info about the file that's missing. + +commit 66b3a3a546b71d485d723c678af1d50d918855aa +Author: Adrien Grand +Date: Tue Mar 12 10:16:41 2019 +0100 + + Warn about the fact that the terms index is moving off-heap. (#39918) + + Lucene 8.0 includes a [change](https://issues.apache.org/jira/browse/LUCENE-8635) + that moves the terms index off-heap for all fields but ID fields. I'm + including this in the migration notes so that users who have queries that match + lots of terms won't be surprised in case of slowdown. + +commit 675489d54c2f1fdd1ea5744ccbdb68e58b757ed9 +Author: Przemyslaw Gomulka +Date: Tue Mar 12 09:19:44 2019 +0100 + + Date_math_expression doc to inform about format backport#39840 #39903 + + The format since 7 should be compatible with java-time. Users should be informed about this on this page + backport#39840 + +commit 466864710ab352cc38249656e48bf48e97663627 +Author: Adrien Grand +Date: Tue Mar 12 08:23:53 2019 +0100 + + Update the how-to section of the docs for 7.0: (#37717) + + - new `rank_feature`/`script_score` queries + - new `index_phrases`/`index_prefixes` options + - disabling `_field_names` doesn't help anymore + - adaptive replica selection is on by default + +commit 5528554aac16cdf87dc1329dee130bbf6671d606 +Author: Adrien Grand +Date: Tue Mar 12 08:10:18 2019 +0100 + + Generate release notes for 7.0.0-alpha1. (#39899) + + We were missing release notes for 7.0.0-alpha1. I generated them by running + the release-notes script with a quick hack that filtered out pull requests + that had been closed on or after 2018-11-15. + + Some breaking changes had been documented in the release notes rather than + the migration guide so I moved them. + +commit 34006105aebdb7c4c44dce2655aa281fa1b064d7 +Author: Daniel Mitterdorfer +Date: Tue Mar 12 06:46:51 2019 +0100 + + Allow IntegTestClusters to run with FIPS 140 JVMs (#39917) (#39923) + + The changes in #39732 mean that nodes in the IntegTest clusters will + now run with whichever java version is defined as `runtime.java` and + not JAVA_HOME anymore. + This means that these nodes will also run in JVM with fips approved + mode enabled and as such, need to have access to the password for the + BCFKS keystore that is used as the default keystore/truststore. + + This change sets the two necessary system properties. + + Resolves #39855 + +commit ce5f09ab0437542013544d09f663c459f874aae8 +Author: Nhat Nguyen +Date: Mon Mar 11 20:51:32 2019 -0400 + + Enforce retention leases require soft deletes (#39922) + + If a primary on 6.7 and a replica on 5.6 are running more than 5 minutes + (retention leases background sync interval), the retention leases + background sync will be triggered, and it will trip 6.7 node due to the + illegal checkpoint value. We can fix the problem by making the returned + checkpoint depends on the node version. This PR, however, chooses to + enforce retention leases require soft deletes, and make retention leases + sync noop if soft deletes is disabled instead. + + Closes #39914 + +commit bf814357ade8c36c8ac0216374aa61176973a760 +Author: Nhat Nguyen +Date: Mon Mar 11 22:10:23 2019 -0400 + + Enable soft deletes in RetentionLeaseIT + + Relates #39922 + +commit 2f47e3d05aee9b3e2e5103e5640ca40784338ff8 +Author: Igor Motov +Date: Fri Mar 8 10:24:52 2019 -0500 + + SQL: values in datetime script aggs should be treated as long (#39773) + + When a query is translated into script terms agg where key has a date + type, it should generate a terms agg with value_type long instead of + date, otherwise the key gets formatted as a string, which confuses + hit extractor. + + Fixes #37042 + +commit 9eb4614fa6e93ab0323783edfa5ba1f41ddecf1d +Author: Armin Braun +Date: Mon Mar 11 19:27:08 2019 +0100 + + More Verbose Assertion in testSnapshotWithStuckNode (#39893) (#39928) + + * The test failure in #39852 is caused by a file in the initial repository when there should not be any + * It seems that on a normal consistent file system no left-over file should exist ever here after the validation finishes and I can't reproduce or see any other path to a dangling file in the fresh respository + => added a more verbose and strict assertion that will log what file is left over next time + * Relates #39852 + +commit b0b0f66669920cd6a723f54369428261d27326b4 +Author: Jake Landis +Date: Mon Mar 11 13:17:27 2019 -0500 + + Remove types from internal monitoring templates and bump to api 7 (#39888) (#39926) + + This commit removes the "doc" type from monitoring internal indexes. + The template still carries the "_doc" type since that is needed for + the internal representation. + + This change impacts the following templates: + monitoring-alerts.json + monitoring-beats.json + monitoring-es.json + monitoring-kibana.json + monitoring-logstash.json + + As part of the required changes, the system_api_version has been + bumped from "6" to "7" and support for version "2" has been dropped. + + A new empty pipeline is now introduced for the version "7", and + the formerly empty "6" pipeline will now remove the type and re-direct + the request to the "7" index. + + Additionally, to due to a difference in the internal representation + (which requires the inclusion of "_doc" type) and external representation + (which requires the exclusion of any type) a helper method is introduced + to help convert internal to external representation, and used by the + monitoring HTTP template exporter. + + Relates #38637 + +commit 4f941c69639661ba85e2adbc980306c1232ee69c +Author: Yannick Welsch +Date: Mon Mar 11 18:59:31 2019 +0100 + + Do not swallow exceptions in TimedRunnable (#39856) + + Executors of type fixed_auto_queue_size (i.e. search / search_throttled) wrap runnables into + TimedRunnable, which is an AbstractRunnable. This is dangerous as it might silently swallow + exceptions, and possibly miss calling a response listener. While this has not triggered any failures in + the tests I have run so far, it might help uncover future problems. + + Follow-up to #36137 + +commit 292eb8b0013658e32511185500b3c8c79ee78f71 +Author: Yannick Welsch +Date: Mon Mar 11 18:58:46 2019 +0100 + + Fix CoordinatorTests.testIncompatibleDiffResendsFullState (#39345) + + This test started failing since decreasing the leader and follower check timeouts (#38298). The + reason is that the test was relying on the default publication timeout to come into effect before + leader / follower check timeouts, which is now not always true anymore. + + Closes #38867 + +commit ca78e44006ffca35eab6022be419d67cd0cbacf8 +Author: Jack Conradson +Date: Mon Mar 11 10:43:03 2019 -0700 + + Fix Painless def [char] to String casts (#39759) + + * Start to fix def char casts. + + * Fix def char to String casts + +commit 31e6f6cf488302579769c8f75f74596b4d2a11fb +Author: Jack Conradson +Date: Fri Mar 8 11:15:12 2019 -0800 + + Add char tests and fix String to char cast (#39725) + + This fixes a bug where a String to char cast in Painless could be done implicitly. + It is now required that a String to char cast is explicit as documented in the + existing specification. This also adds char and Character casting tests. + +commit ea16015bc4e53952da4c40cbbe2d9e80f9733116 +Author: Lisa Cawley +Date: Mon Mar 11 10:38:49 2019 -0700 + + [DOCS] Fixes breaking changes for low level client (#39930) + +commit 7c7933b0aa82ed756c93dc18fee930e369153852 +Author: MK Swanson +Date: Mon Mar 11 12:36:43 2019 -0400 + + [DOCS] Edited intro sentence for clarity. Closes #39865 (#39868) + +commit d30848eb23353121d80cc93c83d2c8613d41d541 +Author: Hendrik Muhs +Date: Mon Mar 11 17:26:16 2019 +0100 + + change internal index to index doc_type, id, source and dest (#39913) + + change internal index to index doc_type, id, source and dest + +commit dd77899278729f1318ad6f1c5c826583ff745e37 +Author: Tim Brooks +Date: Mon Mar 11 10:04:59 2019 -0600 + + Log send failure at debug level if channel closed (#39807) + + Currently we log exceptions due to channel close at the debug level in + the normal exception handler. Currently we log all send failures due to + channel close at the warn level. This commit changes that to only log at + warn if the send failure is not due to channel closed. Additionally, it + adds the ssl engine closed as a channel close exception. + +commit 92a87a45bf38b462f9a2d877fc25354e92e2104b +Author: Costin Leau +Date: Mon Mar 11 17:13:25 2019 +0200 + + SQL: Wrap ZonedDateTime parameters inside scripts (#39911) + + Painless allows ZonedDateTime objects to be passed natively to scripts + which creates problematic translate queries as the ZonedDateTime is + passed as a string instead. + Wrap this with a dedicated method to perform the conversion. + + Fix #39877 + + (cherry picked from commit 4957cad5bda77257d10430ac102e93f5e062148a) + +commit b7be724e50c88540159565ca3859762b9ad9a44e +Author: Yannick Welsch +Date: Mon Mar 11 15:32:58 2019 +0100 + + Check term earlier in publication process (#39909) + + in order to avoid tripping assertPreviousStateConsistency. + + Closes #39314 + +commit 368b5482facc6a9483ab3b4ccc507ad7f366cd6e +Author: Yannick Welsch +Date: Mon Mar 11 15:36:41 2019 +0100 + + Add note about cluster state diffs (#39847) + + Mentions cluster state diffs in CS publishing docs. + +commit 48788269b0470d3a5d04e8636d2c2a5817dee669 +Author: David Kyle +Date: Mon Mar 11 14:02:50 2019 +0000 + + [ML] Correct small inconsistencies in ml APIs spec and docs (#39907) + +commit 6e4f304f889354edf04d5e3feb26a86694d4931a +Author: David Turner +Date: Mon Mar 11 12:00:56 2019 +0000 + + Synchronize pendingOutgoingJoins (#39900) + + Today we use a ConcurrentHashSet to track the in-flight outgoing joins in the + `JoinHelper`. This is fine for adding and removing elements but not for the + emptiness test in `isJoinPending()` which might return false if one join + finishes just after another one starts, even though joins were pending + throughout. + + As used today this is ok: it means the node was trying to join a master but + this join attempt just finished unsuccessfully, and causes it to (rightfully) + reject a `FollowerCheck` from the failed master. However this kind of API + inconsistency is trappy and there is no need to be clever here, so this change + replaces the set with a `synchronizedSet()`. + +commit a079b9fd6d9fa83b52cafa04af430447531d62b4 +Author: Costin Leau +Date: Mon Mar 11 11:50:10 2019 +0200 + + SQL: ConstantProcessor can now handle NamedWriteable (#39876) + + Enhance ConstantProcessor to properly serialize complex objects + (Intervals) that have their own custom serialization/deserialization + mechanism + + Fix #39875 + + (cherry picked from commit ed8a1f9340673e69a44ea7a89679cadb4762e43d) + +commit 471aa6a16ac3b08a3e0c486effedd14f5992497a +Author: Ankit Jain +Date: Fri Mar 8 08:08:55 2019 -0800 + + Fixing 503 Service Unavailable errors during fetch phase (#39086) + + When ESRejectedExecutionException gets thrown on the coordinating node while trying to fetch hits, the resulting exception will hold no shard failures, hence `503` is used as the response status code. In that case, `429` should be returned instead. Also, the status code should be taken from the cause if available whenever there are no shard failures instead of blindly returning `503` like we currently do. + + Closes #38586 + +commit 8925a2c6c2bf998b492e74bee5a8afd326fe8411 +Author: Martijn van Groningen +Date: Mon Mar 11 09:46:42 2019 +0100 + + Further tweak AutoFollowIT#testAutoFollowManyIndices: + * reduce the number of leader indices to be auto followed + * also check the number of follower indices being created + * also check the whether leader indices are marked as auto followed + + Relates to #36761 + +commit 1bc31aca033f8a7507115c5ce5a018edcb7b17f5 +Author: Daniel Mitterdorfer +Date: Mon Mar 11 08:47:51 2019 +0100 + + Mute CcrRetentionLeaseIT#testRetentionLeaseRenewalIsCancelledWhenFollowingIsPaused (#39897) + + Relates #39509 + +commit b841de2e38c6e91ca94db2ce38ec701d8cd0ae7c +Author: Adrien Grand +Date: Mon Mar 11 07:58:28 2019 +0100 + + Don't emit deprecation warnings on calls to the monitoring bulk API. (#39805) (#39838) + + The monitoring bulk API accepts the same format as the bulk API, yet its concept + of types is different from "mapping types" and the deprecation warning is only + emitted as a side-effect of this API reusing the parsing logic of bulk requests. + + This commit extracts the parsing logic from `_bulk` into its own class with a + new flag that allows to configure whether usage of `_type` should emit a warning + or not. Support for payloads has been removed for simplicity since they were + unused. + + @jakelandis has a separate change that removes this notion of type from the + monitoring bulk API that we are considering bringing to 8.0. + +commit 2bbef677700142fcd68e9d2f8846976d5ac95a72 +Author: Adrien Grand +Date: Mon Mar 11 07:58:01 2019 +0100 + + Propagate exceptions in o.e.common.io.Streams. (#39042) (#39848) + + This commit propagates some exceptions that were previously swallowed and also + makes sure that exceptions closing streams are either propagated if the try + block succeeded or added as suppressed exceptions otherwise. + +commit 4da04616c9db51469625469ef7f9b51d4545a6e1 +Author: Benjamin Trent +Date: Sun Mar 10 14:54:02 2019 -0500 + + [ML] refactoring lazy query and agg parsing (#39776) (#39881) + + * [ML] refactoring lazy query and agg parsing + + * Clean up and addressing PR comments + + * removing unnecessary try/catch block + + * removing bad call to logger + + * removing unused import + + * fixing bwc test failure due to serialization and config migrator test + + * fixing style issues + + * Adjusting DafafeedUpdate class serialization + + * Adding todo for refactor in v8 + + * Making query non-optional so it does not write a boolean byte + +commit b5ed03916073b2cc53a60e7ec73f071047bdad9d +Author: David Emanuel Buchmann +Date: Sun Mar 10 16:03:16 2019 +0100 + + plugins/repository-gcs: Update google-cloud-storage/core to 1.59.0 (#39748) + + * plugins/repository-gcs: Update google-cloud-storage / + google-cloud-core to 1.59.0 + + * plugins: Update sha1 for google-cloud-core & google-cloud-storage + +commit 6c6549fc5158f5d0fef8830259ce802048d43044 +Author: Benjamin Trent +Date: Sat Mar 9 14:03:57 2019 -0600 + + [Data-Frame] make the config be strictly parsed on _preview (#39713) (#39873) + + * [Data-Frame] make the config be strictly parsed on _preview + + * adding test to verify strictly parsing + + * adjusting test after master merge + +commit 8454cfc1b26c38dda5050ef6fd898e41b768a7b4 +Author: Julie Tibshirani +Date: Fri Mar 8 15:58:32 2019 -0800 + + Move validation from FieldTypeLookup to MapperMergeValidator. (#39814) + + This commit consolidates more mapping validation logic into the same class. + `FieldTypeLookup` is now a bit simpler, and has the sole responsibility of quickly + resolving field names to their types. + + I have a broader refactor planned around mapping merge validation, but this + change should at least be a step in the right direction. + +commit 993182e426e3c73c287386abed7bb1ce0af2adf9 +Author: Nhat Nguyen +Date: Thu Mar 7 12:07:38 2019 -0500 + + Combine overriddenOps and skippedOps in translog (#39771) + + These two stats are not important enough to be distinguishable. + This change combines them into a single stat. + + Closes #33317 + +commit 671a209ed923511e3b3f29eecb277aa705d8570f +Author: Mayya Sharipova +Date: Fri Mar 8 16:16:03 2019 -0500 + + Correct errors in min_hash filter documentation + + Related to #39671 + +commit be9c37fc76b6d38e4edcad652d2752355c2b1e75 +Author: Julie Tibshirani +Date: Thu Mar 7 14:11:27 2019 -0800 + + Small simplifications to mapping validation. (#39777) + + These simplifications to `MapperMergeValidator` are possible now that there is + always a single mapping definition. + + * Remove the type argument in `validateMapperStructure`. + * Remove unnecessary checks against existing mappers. + +commit a0a91f74ffb132165a08beb982f9577f3293c21b +Author: Nhat Nguyen +Date: Fri Mar 8 12:53:27 2019 -0500 + + Treat TransportService stopped error as node is closing (#39800) + + If TransportService is stopped before a shard-failure request is sent + but after the request is registered, TransportService will notify + ReplicationOperation a TransportException with an error message: + "transport stop, action: internal:cluster/shard/failure". + + Relates #39584 + +commit 6c52da54c854cb14b170a87ee54786b7c14cb653 +Author: Tal Levy +Date: Fri Mar 8 12:15:32 2019 -0800 + + fix index refresh in test within 20_mix_typeless_typeful (#39198) (#39804) + + the test "Implicitly create a typeless ... typed template" + fails occasionally because the index operation hasn't + propogated to update the index mapping in time for the + following assertion about a dynamically mapped field "bar". + + error failed with: + + ``` + field [test-1.mappings.my_type.properties.bar] doesn't have a true value + Expected: not null + but: was null + ``` + + refreshing the index should resolve this timing issue. + +commit 5612ed97caba71f614592c067bd003f41494044e +Author: Tim Brooks +Date: Fri Mar 8 12:50:24 2019 -0700 + + Add log warnings for long running event handling (#39729) + + Recently we have had a number of test issues related to blocking + activity occuring on the io thread. This commit adds a log warning for + when handling event takes a >150 milliseconds. This is implemented + for the MockNioTransport which is the transport used in + ESIntegTestCase. + +commit 73a672b8dda76718d6727a1652bd3ce67a6dde6e +Author: Jason Tedor +Date: Fri Mar 8 14:51:39 2019 -0500 + + Fix Watcher stats class cast exception (#39821) + + The watcher stats implementation tries to look at all queued watches + before preparing the result. We want to cast these to a + WatchExecutionTask to extract the context to prepare the stats for + queued watches. The problem is that not all tasks on the watcher queue + were WatchExecutionTask. This is because a manually executed watch was + not even at all wrapped in a WatchExecutionTask. Moreover, we were using + ExecutorService#submit(Runnable) which would wrap the Runnable in a + FutureTask. This commit addresses this by using a WatchExecutionTask, + and also using ExecutorService#execute(Runnable) so that no wrapping + occurs. This will let us continue with the assumption that all queued + tasks are WatchExecutionTasks. + +commit 3246d3e355721c49d13dc0c946b2f4ae7f6d26b2 +Author: lcawl +Date: Fri Mar 8 11:13:02 2019 -0800 + + [DOCS] Adds tagged region for notable breaking changes + +commit 35dd52461f7ed5857d70491e9e089bd0e8e4df13 +Author: Ryan Ernst +Date: Wed Mar 6 10:11:34 2019 -0800 + + Make integ tests always use runtime java (#39732) + + This commit ensures cluster formation always uses runtime java, never + attempting to use the bundled jdk, since the integ test zip does not + contain it. + +commit 89a1f155d3a8e26f110639d8945c9553a3f6dc99 +Author: David Roberts +Date: Thu Mar 7 18:53:44 2019 +0000 + + Give jspawnhelper execute permissions in bundled JDK (#39787) + + The posix_spawn method of launching a process from Java + goes via an intermediate process called jspawnhelper + which lives in the lib directory rather than the bin + directory and hence got missed by the original chmod + loop. This change adds jspawnhelper as a special case. + It's the only program that's in the lib directory in a + macOS JDK 11. + +commit 465343f12a54111709ca7c7cd776edcaba1e0c2b +Author: Ryan Ernst +Date: Fri Mar 8 11:04:18 2019 -0800 + + Bundle java in distributions (#38013) + + * Bundle java in distributions + + Setting up a jdk is currently a required external step when installing + elasticsearch. This is particularly problematic for the rpm/deb packages + as installing a jdk in the same package installation command does not + guarantee any order, so must be done in separate steps. Additionally, + JAVA_HOME must be set and often causes problems in selecting a correct + jdk when, for example, the system java is an older unsupported version. + + This commit bundles platform specific openjdks into each distribution. + In addition to eliminating the issues above, it also presents future + possible improvements like using jlink to build jdk images only + containing modules that elasticsearch uses. + + closes #31845 + +commit e0abc3ce96a1ba60dba5f578e173e2046187c336 +Author: Jake Landis +Date: Fri Mar 8 12:46:36 2019 -0600 + + Remove the index type from internal watcher indexes (#39761) (#39853) + + This commit removes the "doc" type from watcher internal indexes. + The template still carries the "_doc" type since that is needed for + the internal representation. + + This impacts the .watches, .triggered-watches, and .watch-history indexes. + + External consumers do not need any changes since all external calls + go through the _watcher API, and should not interact with the the .index directly. + + Relates #38637 + +commit 3c7fafd0cc4fb6deba9de903d69c2d0b25f6e18b +Author: Albert Zaharovits +Date: Fri Mar 8 20:18:00 2019 +0200 + + Fix token invalidation when retries exhausted (#39799) + + Fixes an error about missing to call the index invalidation listener + when retry count is exhausted but there are still tokens to be retried. + +commit 6675bafc490c7bd62ecdb943b8f522f4d91b28f3 +Author: Jason Tedor +Date: Fri Mar 8 12:24:47 2019 -0500 + + Simplify CcrRetentionLeaseIT#testForgetFollower + + This test was more complicated than necessary, where we were capturing + requests to prevent removal of retention leases, so that our forget + follower request could remove the retention leases instead. Instead, a + pause is enough to ensure that the retention leases are not re-added + after we remove them by the forget follower request. This commit + simplifies this test, and should remove some spurious failures. + + Relates #39850 + +commit c23ff0803edd1588649499844a0d9935d111b765 +Author: Przemyslaw Gomulka +Date: Fri Mar 8 16:34:17 2019 +0100 + + Fix JsonLoggerTests.testJsonInStacktraceMessageIsSplitted backport#39830 #39834 + + a test was splitting a json log message with a wrong line separator. + Using System.lineSeparator consistently prevents this problem + closes #39829 + backport#39830 + +commit e6b9262a3123a923449abaf05d71e536403ee3c7 +Author: Gordon Brown +Date: Fri Mar 1 11:11:17 2019 -0700 + + Mute testOpenCloseApiWildcards (#39578) (#39579) + +commit a8530c55314ae9acb99998638db3247ff00de98d +Author: Jake Landis +Date: Fri Mar 8 08:23:30 2019 -0600 + + Update logstash-management.json to use typeless template (#38653) (#39819) + + This commit changes the type from "doc" to "_doc" for the + .logstash-management template. Since this is an internally + managed template it does not always go through the REST + layer for it's internal representation. The internal + representation requires the default "_doc" type, which for + external templates is added in the REST layer. + + Related #38637 + +commit aec2db78eafa3447e86528b65f3fb296fd29e137 +Author: David Roberts +Date: Fri Mar 8 13:28:27 2019 +0000 + + Mute RareClusterStateIT.testDelayedMappingPropagationOnReplica + + Due to https://github.com/elastic/elasticsearch/issues/36813 + +commit 6c2e831e948e9d4daca0c732ab5c797403ef0d1f +Author: David Kyle +Date: Fri Mar 8 12:18:55 2019 +0000 + + [ML-Dataframe] Data frame config HLRC objects (#39825) + +commit 8666aa1ed28f44d8d848aa7bba0d377fe792fa64 +Author: Martijn van Groningen +Date: Fri Mar 8 12:42:32 2019 +0100 + + unmuted and tweaked test + + Relates to #36761 + +commit 366eef99a18563b7b4cff19b00ffcf71a2d4977e +Author: David Roberts +Date: Fri Mar 8 11:42:13 2019 +0000 + + Mute SharedClusterSnapshotRestoreIT.testCloseOrDeleteIndexDuringSnapshot + + Due to https://github.com/elastic/elasticsearch/issues/39828 + +commit 50d742320d34bc2ab533428695aeb48216f7713b +Author: Hendrik Muhs +Date: Fri Mar 8 12:15:09 2019 +0100 + + store the doc type in the internal index (#39824) + + store the doc type in the internal data frame index + +commit 5d68143b184723d486227daaa0aabbc2e0bb1dce +Author: David Turner +Date: Fri Mar 8 10:01:36 2019 +0000 + + Reformat elasticsearch-node messages (#39811) + + Flows the warning messages emitted by the `elasticsearch-node` tool to a width + of 72 characters and tweaks the wording slightly. + +commit 1520ddc8b68e83b93d80aab0fd312213b6657a23 +Author: CJ Cenizal +Date: Thu Mar 7 14:08:47 2019 -0800 + + Update distribution build instructions to reflect file names with OS/architecture classifiers. (#39762) (#39812) + +commit 797d6b8a666e18c6724c21812fb2fc7b39788776 +Author: Jake Landis +Date: Thu Mar 7 13:31:41 2019 -0600 + + Execute ingest node pipeline before creating the index (#39607) (#39796) + + Prior to this commit (and after 6.5.0), if an ingest node changes + the _index in a pipeline, the original target index would be created. + For daily indexes this could create an extra, empty index per day. + + This commit changes the TransportBulkAction to execute the ingest node + pipeline before attempting to create the index. This ensures that the + only index created is the original or one set by the ingest node pipeline. + This was the execution order prior to 6.5.0 (#32786). + + The execution order was changed in 6.5 to better support default pipelines. + Specifically the execution order was changed to be able to read the settings + from the index meta data. This commit also includes a change in logic such + that if the target index does not exist when ingest node pipeline runs, it + will now pull the default pipeline (if one exists) from the settings of the + best matched of the index template. + + Relates #32786 + Relates #32758 + Closes #36545 + +commit 8ec456b5df1fc724dab6e0342873c3c924780730 +Author: Lee Hinman +Date: Thu Mar 7 11:26:36 2019 -0700 + + Maintain step order for ILM trace logging (#39522) + + When trace logging is enabled we log the computed steps for a policy. This + commit makes sure that the steps that are logged are in the same order they will + be run when the policy executes. This makes it much easier to reason about the + policy if the move-to-step API is ever required in the future. + +commit 4d41310be567db0e01a2d32461444672a937e4be +Author: Hendrik Muhs +Date: Thu Mar 7 19:22:34 2019 +0100 + + [ML-DataFrame] fix wire serialization issues in data frame response objects (#39790) + + fix wire serialization issues in data frame response objects + +commit af4e740500726d36a527fad7242666c9f0640760 +Author: Martijn Laarman +Date: Thu Mar 7 11:34:01 2019 +0100 + + Document scroll param on reindex.json (#38615) + + The Reindex API also exposes `scroll` as a querystring parameter. + +commit 3dfd62cf5181c2e379fc19b4d8789ba0cc6c4283 +Author: Martijn van Groningen +Date: Thu Mar 7 17:55:28 2019 +0100 + + unmute test #39355 + +commit 8043fefcf655068c2cdc0e120354435fba6753c2 +Author: Tim Brooks +Date: Thu Mar 7 09:36:28 2019 -0700 + + Log close_notify during handshake at debug level (#39715) + + A TLS handshake requires exchanging multiple messages to initiate a + session. If one side decides to close during the handshake, it is + supposed to send a close_notify alert (similar to closing during + application data exchange). The java SSLEngine engine throws an + exception when this happens. We currently log this at the warn level if + trace logging is not enabled. This level is too high for a valid + scenario. Additionally it happens all the time in tests (quickly closing + and opened transports). This commit changes this to be logged at the + debug level if trace is not enabled. Additionally, it extracts the + transport security exception handling to a common class. + +commit b9586f62cc307b42bc5e006e19d459747f5f9b61 +Author: Jason Tedor +Date: Thu Mar 7 11:43:26 2019 -0500 + + Fix CCR HLRC docs + + This commit fixes the CCR HLRC docs by including the forget follower API + docs in the HLRC docs. + +commit 0250d554b6f5071a4abe220d7147f26e08281d05 +Author: Jason Tedor +Date: Thu Mar 7 10:29:10 2019 -0500 + + Introduce forget follower API (#39718) + + This commit introduces the forget follower API. This API is needed in cases that + unfollowing a following index fails to remove the shard history retention leases + on the leader index. This can happen explicitly through user action, or + implicitly through an index managed by ILM. When this occurs, history will be + retained longer than necessary. While the retention lease will eventually + expire, it can be expensive to allow history to persist for that long, and also + prevent ILM from performing actions like shrink on the leader index. As such, we + introduce an API to allow for manual removal of the shard history retention + leases in this case. + +commit 6c75a2f2b017107c1c5f4a56ab118ed26f39c789 +Author: Alpar Torok +Date: Thu Mar 7 17:48:49 2019 +0200 + + Testclusters: start using it for testing some plugins (#39693) + + * enable testclusters for some plugins + +commit 7dcc191aa8383a438702567190d4b40ff52f903e +Author: Alpar Torok +Date: Thu Mar 7 13:54:20 2019 +0200 + + Fix verify versions (#39624) + + closes #38708 + +commit 0f89427eb6aec9f346d42fba590cb7b5e0d1b9af +Author: Alpar Torok +Date: Thu Mar 7 17:25:09 2019 +0200 + + Back port build changes from same version bwc tests (#39744) + + * Back port build changes from #39102 + + This back-ports how versions are determined and bwc test are set up from + #39102 without enabling the bwc from current version tests so it's + easier/possible to backmerge future buld changes. + It's expected that the tets are lacking many of the required fixes in + this version to enable them. + +commit 54d41afac178f6f27d1b9aea87899bbcd20b4eff +Author: Mayya Sharipova +Date: Thu Mar 7 08:47:32 2019 -0500 + + Add documentation for min_hash filter (#39671) + + Closes #20757 + +commit 4df26a2a907ce8a6f5d630940aa980a24f073610 +Author: Akshesh Doshi +Date: Thu Mar 7 20:27:24 2019 +0700 + + Implement ConcatFilesTask from Groovy to Java issues#34459 (#37497) + + relates #344597 + +commit 6c19d872a01edf1225fc9ba5847a4eb10e6b6136 +Author: Ioannis Kakavas +Date: Thu Mar 7 12:59:23 2019 +0200 + + Fix testRefreshingMultipleTimesWithinWindowSucceeds (#39701) + + Previously all the threads were writing the received tokens to a + HashSet. In cases with many threads, sometimes (1 every ~25 tests) + calling size() on the HashSet returned 2 even though it seemed to + contain only one String and there was no evidence from logging that + threadSecurityClient.refreshToken() ever returned a different + access or refresh token. + + This commit changes the test to use a ConcurrentHashMap instead, + checking that we only received one pair of access token/refresh token + eventually. It also adds a check so that we won't take into consideration + tokens that are returned after 30s, hence not in the concurrent refresh + time window. + +commit 95bed811985154cd044e6b878a3ee9b298bc952d +Author: Przemyslaw Gomulka +Date: Thu Mar 7 12:06:18 2019 +0100 + + Change licence expiration date pattern Backport(#39681) #39781 + + Due to migration from joda to java.time licence expiration 'full date' format + has to use 4-char pattern (MMMM). Also since jdk9 the date with ROOT + locale will still return abbreviated days and month names. + + closes #39136 + backport #39681 + +commit 213cc6673c4c85e95edc6c9c890abeb697c8c0c5 +Author: Armin Braun +Date: Thu Mar 7 08:31:46 2019 +0100 + + Remove Dead Code in o.e.util package (#39717) (#39779) + + * None of this code is used so we should delete it, we can always bring it back if needed + +commit 34ea84948c0c9041b74403b9b4d4986c46133714 +Author: Alpar Torok +Date: Thu Mar 7 09:07:11 2019 +0200 + + Fix bwc tests failure to extract (#39619) + + * Set correct packaging for older versions + + Continue using zip packages for pre 7 + Some other bwc fixes that hid behind this one. + Closes #39441 #39751 + +commit f5da028a3d1289e29c941d703cfa9fce3e761090 +Author: Armin Braun +Date: Thu Mar 7 07:24:08 2019 +0100 + + Chunk + Throttle Netty Writes (#39286) (#39778) + + * Chunk large writes and throttle on a non-writable channel to reduce direct memory usage by Netty + +commit 83688ce2d45bd8ba1228c5394270abb93d5a436f +Author: Nhat Nguyen +Date: Wed Mar 6 21:48:54 2019 -0500 + + Unmute testFollowIndexAndCloseNode + + Resolved in #39584 + +commit bb2f8485f15f9c902bd22bec7fc4f9df8ebd87db +Author: Armin Braun +Date: Wed Mar 6 21:48:17 2019 +0100 + + Wipe Snapshots Before Indices in RestTests (#39662) (#39763) + + * Wipe Snapshots Before Indices in RestTests + + * If we have a snapshot ongoing from the previous test and enter this method, then deleting the indices fails, which in turn fails the whole wipe + * Fixed by first deleting/aborting snapshots + +commit b69affda6adadc6c191c2e0835343fa7d91d21c6 +Author: Nhat Nguyen +Date: Wed Mar 6 14:59:41 2019 -0500 + + Use unwrapped cause to determine if node is closing (#39723) + + We need to unwrap and use the actual cause when determining if the node + with primary shard is shutting down because TransportService will throw + a TransportException wrapped in a SendRequestTransportException. + + Relates #39584 + +commit 1fe7cb594fb11b344433fda951dedc833acff140 +Author: Nhat Nguyen +Date: Tue Mar 5 11:22:03 2019 -0500 + + Don’t ack if unable to remove failing replica (#39584) + + Today when a replicated write operation fails to execute on a replica, + the primary will reach out to the master to fail that replica (and mark + it stale). We then won't ack that request until the master removes the + failing replica; otherwise, we will lose the acked operation if the + failed replica is still in the in-sync set. However, if a node with the + primary is shutting down, we might ack such request even though we are + unable to send a shard-failure request to the master. This happens + because we ignore NodeClosedException which is triggered when the + ClusterService is being closed. + + Closes #39467 + +commit 3591da6ff89019b31bba40bcc642858ae7c7dbfc +Author: Nhat Nguyen +Date: Tue Mar 5 10:11:18 2019 -0500 + + Simplify FrozenEngine#getReader (#39539) + + We really don’t need a try/finally in this method. + +commit 7da62d3b798c9bf86c3aa96f257c90256f512972 +Author: Ryan Ernst +Date: Wed Mar 6 12:17:12 2019 -0800 + + Add note about negative timestamps to migration guide (#39734) + + This commit adds a note to the migration guide for 7.0 about negative + epoch timestamps no longer being supported. + + closes #39375 + +commit 1873de5240c628d96e963f038796a189a3c501ec +Author: markharwood +Date: Wed Mar 6 19:02:04 2019 +0000 + + Bug fix for AnnotatedTextHighlighter - port of 39525 (#39749) + + Bug fix for AnnotatedTextHighlighter - port of 39525 + + Relates to #39395 + +commit d094107592183ca601e2e8402c4e12266f64658b +Author: Yannick Welsch +Date: Wed Mar 6 17:51:23 2019 +0100 + + Fix SharedClusterSnapshotRestoreIT + + Relates to #39644 + +commit fef11f7efc6f94ce750b106ccb5aae75cca14cbd +Author: Yannick Welsch +Date: Wed Mar 6 14:47:41 2019 +0100 + + Allow snapshotting replicated closed indices (#39644) + + This adds the capability to snapshot replicated closed indices. + + It also changes snapshot requests in v8.0.0 to automatically expand wildcards to closed indices and hence start snapshotting closed indices by default. For v7.1.0 and above, wildcards are by default only expanded to open indices, which can be changed by explicitly setting the expand_wildcards option either to all or closed. + + Note that indices are always restored as open indices, even if they have been snapshotted as closed replicated indices. + + Relates to #33888 + +commit e620fb2e4ac71e153d6d09e3959ac2a715378d66 +Author: Simon Willnauer +Date: Wed Mar 6 15:28:20 2019 +0100 + + Add option to force load term dict into memory (#39741) + + Lucene added an optimization to leave the term dictionary on disk + for non-id like fields. This change happened very late in the release + processes such that it's better to have an escape hatch if certain + use-cases are hurt by this optimization. This setting might be + removed in the future if it turns out to be unnecessary. + +commit fb1005fffcf9d0221c7d13f3f52f0784520b74ff +Author: Albert Zaharovits +Date: Wed Mar 6 15:32:23 2019 +0200 + + Fix Token Service retry mechanism (#39639) + + Fixes several errors of the token retry logic: + + * not checking for backoff.hasNext() before calling backoff.next() + * checking for backoff.hasNext() without calling backoff.next() + * not preserving the context on the retry + * calling scheduleWithFixedDelay instead of schedule + +commit 6c503824c8c586bc436152e598ec34edfe26cd53 +Author: Christoph Büscher +Date: Wed Mar 6 14:17:26 2019 +0100 + + Fix occasional SearchServiceTests failure (#39697) + + Currently SearchServiceTests.testCloseSearchContextOnRewriteException can fail + if a refresh happens while we test for the SearchPhaseExecutionException that is + thrown later in the test. The test takes the current Store#refCount and expects + it to be the same after the exception is thrown. If a refresh happens in that + interval however, the refCound will be different, causing the test to fail. This + can be provoked e.g. by running this section in a tight loop. + Switching of refresh for this tests solves the issue. + +commit 5f8f91c03bc796268d90c5a377a92073f46b65c6 +Author: David Roberts +Date: Wed Mar 6 12:29:34 2019 +0000 + + [ML] Use scaling thread pool and xpack.ml.max_open_jobs cluster-wide dynamic (#39736) + + This change does the following: + + 1. Makes the per-node setting xpack.ml.max_open_jobs + into a cluster-wide dynamic setting + 2. Changes the job node selection to continue to use the + per-node attributes storing the maximum number of open + jobs if any node in the cluster is older than 7.1, and + use the dynamic cluster-wide setting if all nodes are on + 7.1 or later + 3. Changes the docs to reflect this + 4. Changes the thread pools for native process communication + from fixed size to scaling, to support the dynamic nature + of xpack.ml.max_open_jobs + 5. Renames the autodetect thread pool to the job comms + thread pool to make clear that it will be used for other + types of ML jobs (data frame analytics in particular) + + Backport of #39320 + +commit 52fd102e23929e3a006c822470e256126fb66f19 +Author: Andrey Ershov +Date: Wed Mar 6 13:49:24 2019 +0100 + + Avoid serialising state if it was already serialised (#39179) + + When preparing the state to send to other nodes, we're serializing it + for each node, despite using putIfAbsent. + This commit checks if the state was already serialized for this node + version before performing the potentially expensive computation. + The map is not used by multiple threads, so computeIfAbsent is not + needed (and could not be used here easily, because IOException could + be thrown). + + (cherry picked from commit c99be63b43f5250f3cd220130df73c5e9e097459) + +commit 95409d3a7eca1c155ae327aa7c5cd5f3cb4e7b44 +Author: Ian <30798206+ianhuang92@users.noreply.github.com> +Date: Wed Mar 6 18:14:58 2019 +0800 + + Correct date in daterange-aggregation.asciidoc (#39727) + +commit 295e39a8c841b9eba5b7e8c00e9ebb12695eca38 +Author: David Turner +Date: Wed Mar 6 09:23:41 2019 +0000 + + Drop node if asymmetrically partitioned from master (#39598) + + When a node is joining the cluster we ensure that it can send requests to the + master _at that time_. If it joins the cluster and _then_ loses the ability to + send requests to the master then it should be removed from the cluster. Today + this is not the case: the master can still receive responses to its follower + checks, and receives acknowledgements to cluster state publications, so has no + reason to remove the node. + + This commit changes the handling of follower checks so that they fail if they + come from a master that the other node was following but which it now believes + to have failed. + +commit 77dd71184742b81fb33efcf51caec289a5675987 +Author: David Turner +Date: Wed Mar 6 09:23:12 2019 +0000 + + Tidy up GroupedActionListener (#39633) + + Today the `GroupedActionListener` accepts a `defaults` parameter but all + callers pass an empty list. Also it is permitted to pass an empty group but + this is trappy because the delegated listener is never be called in that case. + This commit removes the `defaults` parameter and forbids an empty group. + +commit c91dcbd5eeee4d6052f2fd9e35df7ffe4b84ef17 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Mar 6 18:53:59 2019 +1100 + + Types removal security index template (#39705) (#39728) + + As we are moving to single type indices, + we need to address this change in security-related indexes. + To address this, we are + - updating index templates to use preferred type name `_doc` + - updating the API calls to use preferred type name `_doc` + + Upgrade impact:- + In case of an upgrade from 6.x, the security index has type + `doc` and this will keep working as there is a single type and `_doc` + works as an alias to an existing type. The change is handled in the + `SecurityIndexManager` when we load mappings and settings from + the template. Previously, we used to do a `PutIndexTemplateRequest` + with the mapping source JSON with the type name. This has been + modified to remove the type name from the source. + So in the case of an upgrade, the `doc` type is updated + whereas for fresh installs `_doc` is updated. This happens as + backend handles `_doc` as an alias to the existing type name. + + An optional step is to `reindex` security index and update the + type to `_doc`. + + Since we do not support the security audit log index, + that template has been deleted. + + Relates: #38637 + +commit aaecaf59a44876288b5bc4fc1fe502179518c180 +Author: Armin Braun +Date: Wed Mar 6 08:13:15 2019 +0100 + + Optimize Bulk Message Parsing and Message Length Parsing (#39634) (#39730) + + * Optimize Bulk Message Parsing and Message Length Parsing + + * findNextMarker took almost 1ms per invocation during the PMC rally track + * Fixed to be about an order of magnitude faster by using Netty's bulk `ByteBuf` search + * It is unnecessary to instantiate an object (the input stream wrapper) and throw it away, just to read the `int` length from the message bytes + * Fixed by adding bulk `int` read to BytesReference + +commit 4a3fa5ac7d8274a79df4e86c2e92ef33c93ec6c1 +Author: Jason Tedor +Date: Tue Mar 5 22:19:19 2019 -0500 + + Remove beta label from CCR (#39722) + + This commit removes the beta label from CCR. + +commit 75a0d4f470cc703a065ba122009aae2fc7425d83 +Author: Jason Tedor +Date: Tue Mar 5 22:02:31 2019 -0500 + + Rename retention lease setting (#39719) + + This commit renames the retention lease setting + index.soft_deletes.retention.lease so that it is under the namespace + index.soft_deletes.retention_lease. As such, we rename the setting to + index.soft_deletes.retention_lease.period. + +commit 504c792861e74842d1dd6e74d8504caa83b18498 +Author: Jason Tedor +Date: Tue Mar 5 10:15:59 2019 -0700 + + Add Docker build type (#39378) + + This commit adds a new build type (together with deb/rpm/tar/zip) to + represent the official Docker images. This build type will be displayed + in APIs such as the main and nodes info APIs. + +commit eb288a6f857d1a16d7ea686ba48cc8de17b0f15b +Author: Gordon Brown +Date: Tue Mar 5 11:45:34 2019 -0700 + + Use any index specified by .watches for Watcher (#39541) (#39708) + + Previously, Watcher only attached its listener to indices that started + with the prefix `.watches`, which causes Watcher to silently fail to + schedule newly created Watches if the `.watches` alias is redirected to + an index that does not start with `.watches`. + + Watcher now attaches the listener to all indices, so that Watcher can + respond to changes in which index has the `.watches` alias. + + Also adjusts the tests to randomly use non-prefixed concrete indices + for .watches and .triggered_watches. + +commit ecb6df137c2975ab7f54156295e32ec6145659ad +Author: jimczi +Date: Tue Mar 5 18:18:45 2019 +0100 + + fix typo in synonym graph filter docs + +commit ee7c01988fc717fad2c8912e2ff53c46d957a111 +Author: Tim Brooks +Date: Tue Mar 5 09:50:58 2019 -0700 + + Add documentation on remote recovery (#39483) + + This is related to #35975. It adds documentation on the remote recovery + process. Additionally, it adds documentation about the various settings + that can impact the process. + +commit fad52acf5acca84d10bda7402a1d8e8012ee67a9 +Author: Tomas Della Vedova +Date: Tue Mar 5 18:12:24 2019 +0100 + + Removed incorrect ML YAML tests (#39400) + + A client cannot know that a job_id is already taken, so + this test should not have been specified as a client test + +commit 9d0211485c32b26cbefc823d0ef37158b13e593f +Author: Luca Cavanna +Date: Tue Mar 5 13:51:56 2019 +0100 + + Tie-break completion suggestions with same score and surface form (#39564) + + In case multiple completion suggestion entries have the same score and + surface form, the order in which such options will be returned is + currently not deterministic. + + With this commmit we introduce tie-breaking for such situations, based + on shard id, index name, index uuid and doc id like we already do for + ordinary search hits. With this change we also make shardIndex + mandatory when sorting and comparing completion suggestion options, + which was previously only needed later when fetching hits). + + Also, we need to make sure shardIndex is properly set when merging + completion suggestions coming from multiple clusters in + `SearchResponseMerger` + +commit ca834085428aa17fe9ef586210e44df325718cd6 +Author: Samuel Cifuentes García <33845232+srensamblador@users.noreply.github.com> +Date: Tue Mar 5 16:17:01 2019 +0100 + + Improved Terms Aggregation documentation (#38892) + + Added a note after the first query example talking about fielddata. + +commit 160dc29f0e99f7ae458d0df8e9c2542da22bf3d4 +Author: Jim Ferenczi +Date: Tue Mar 5 12:03:39 2019 +0100 + + Handle total hits equal to track_total_hits (#37907) + + This change ensures that a total hits equal to the value set for + track_total_hits is not considered as a lower bound. + +commit e94d32d069c153a629f53ef9a7ff218694939573 +Author: David Roberts +Date: Tue Mar 5 14:05:38 2019 +0000 + + Add roles and cluster privileges for data frame transforms (#39661) + + This change adds two new cluster privileges: + + * manage_data_frame_transforms + * monitor_data_frame_transforms + + And two new built-in roles: + + * data_frame_transforms_admin + * data_frame_transforms_user + + These permit access to the data frame transform endpoints. + (Index privileges are also required on the source and + destination indices for each data frame transform, but + since these indices are configurable they it is not + appropriate to grant them via built-in roles.) + +commit 750ec8ba5378ddb0aff9d82a24ecc876d9d58c5d +Author: Armin Braun +Date: Tue Mar 5 15:04:16 2019 +0100 + + Minor Cleanups in QueryPhase (#39680) (#39694) + + * Soften redundant cast to allow use of `DeterministicTaskQueue` in this class for #39504 + * Remove two redundant variables and lower visibility in two possible spots + * Make field `final` + +commit 5cdea6ef17acc4541be6baac62334efcd1983497 +Author: Christoph Büscher +Date: Tue Mar 5 14:30:10 2019 +0100 + + Fix Fuzziness#asDistance(String) (#39643) + + Currently Fuzziness#asDistance(String) doesn't work for custom AUTO values. If + the fuzziness is AUTO, the method returns the correct edit distance to use, + depending on the input string, but for custom AUTO values it currently always + returns an edit distance of 1. Correcting this and adding unit and integration + tests to catch these cases. + + Closes #39614 + +commit 19f6a3535821eb1104f5047073457f6bcd8efc15 +Author: Simon Willnauer +Date: Tue Mar 5 14:09:49 2019 +0100 + + Move BWC Version to 7.1.0 after backport + + Relates to #39512 + +commit d112c890417a2104e9b1e2d0d03364e1f8154787 +Author: Simon Willnauer +Date: Tue Mar 5 14:01:41 2019 +0100 + + Allow inclusion of unloaded segments in stats (#39512) + + Today we have no chance to fetch actual segment stats for segments that + are currently unloaded. This is relevant in the case of frozen indices. + This allows to monitor how much memory a frozen index would use if it was + unfrozen. + +commit 7ed9d52824f466ddfecedc85e73d5bc8827a8fa7 +Author: Ioannis Kakavas +Date: Tue Mar 5 14:55:59 2019 +0200 + + Support concurrent refresh of refresh tokens (#39647) + + This is a backport of #39631 + + Co-authored-by: Jay Modi jaymode@users.noreply.github.com + + This change adds support for the concurrent refresh of access + tokens as described in #36872 + In short it allows subsequent client requests to refresh the same token that + come within a predefined window of 60 seconds to be handled as duplicates + of the original one and thus receive the same response with the same newly + issued access token and refresh token. + In order to support that, two new fields are added in the token document. One + contains the instant (in epoqueMillis) when a given refresh token is refreshed + and one that contains a pointer to the token document that stores the new + refresh token and access token that was created by the original refresh. + A side effect of this change, that was however also a intended enhancement + for the token service, is that we needed to stop encrypting the string + representation of the UserToken while serializing. ( It was necessary as we + correctly used a new IV for every time we encrypted a token in serialization, so + subsequent serializations of the same exact UserToken would produce + different access token strings) + + This change also handles the serialization/deserialization BWC logic: + + In mixed clusters we keep creating tokens in the old format and + consume only old format tokens + In upgraded clusters, we start creating tokens in the new format but + still remain able to consume old format tokens (that could have been + created during the rolling upgrade and are still valid) + When reading/writing TokensInvalidationResult objects, we take into + consideration that pre 7.1.0 these contained an integer field that carried + the attempt count + + Resolves #36872 + +commit e8d97443407c64a0f6f88383f4596bfec66c5aa9 +Author: Armin Braun +Date: Tue Mar 5 12:37:49 2019 +0100 + + Use Threadpool Time in ClusterApplierService (#39679) (#39685) + + * Use threadpool's time in `ClusterApplierService` to allow for deterministic tests + * This is a part of/requirement for #39504 + +commit 380dc27d91bf496bd517b513eefabbc295fad767 +Author: Gordon Brown +Date: Fri Mar 1 14:20:18 2019 -0700 + + Mute testCloseWhileRelocatingShards (#39589) + +commit 0b14782b234994d753c00a543636659804b47b74 +Author: Alan Woodward +Date: Tue Mar 5 10:31:44 2019 +0000 + + Add stopword support to IntervalBuilder (#39637) + + The match interval builder analyses input text and converts it to an IntervalSource, and as such + may generate token streams with stopwords. This commit deals with these by using the extend + factory to cover the gaps produced by these stopwords so that phrase and ordered queries work + correctly. + +commit e7dbfda5d33fe2dd964e483f3c37a5e797969668 +Author: Albert Zaharovits +Date: Tue Mar 5 12:13:25 2019 +0200 + + Fix security index auto-create and state recovery race (#39582) + + Previously, the security index could be wrongfully recreated. This might + happen if the index was interpreted as missing, as in the case of a fresh + install, but the index existed and the state did not yet recover. + + This fix will return HTTP SERVICE_UNAVAILABLE (503) for requests that + try to write to the security index before the state has not been recovered yet. + +commit 5c023770d2740650f6d893e33e5ef411a21d2c39 +Author: Dimitris Athanasiou +Date: Tue Mar 5 12:42:30 2019 +0200 + + [ML] Disable security audit trail in native integ tests suite (#39683) + + Investigating how to make DeleteExpiredDataIT faster, it was + revealed that the security audit trail threads were quite hot. + Disabling that seems to be helping quite a bit with making this + test faster. This commit also unmutes the test to see how it goes + with the audit trail disabled. + + Relates #39658 + Closes #39575 + +commit b78a8a3e80a526c9040c627b43a024da91d30570 +Author: Martijn van Groningen +Date: Tue Mar 5 08:54:56 2019 +0100 + + Use RestToXContentListener in painless execute action rest action. (#39638) + +commit 171ef052127a3a9988510e3171dff6d9f77cb5ac +Author: Daniel Mitterdorfer +Date: Tue Mar 5 07:34:29 2019 +0100 + + Add more diagnostic info when Windows JVM is not found (#33804) (#39652) + + With this commit we provide more info in an existing error message that is + raised when the file `jvm.dll` cannot be found on Windows when installing + Elasticsearch as a service. + +commit 4dadeba039f302b4bfbf04176d8727595fefa554 +Author: lcawl +Date: Mon Mar 4 15:06:00 2019 -0800 + + [DOCS] Sorts security APIs + +commit f494defdbdd6fb2f0019ee4f026090ae57a732dc +Author: Lisa Cawley +Date: Mon Mar 4 12:46:22 2019 -0800 + + [DOCS] Updates API in Watcher transform context (#39540) + +commit 2fe1fa897256cdb30ed2201cb624af3fb9aa961a +Author: Christoph Büscher +Date: Mon Mar 4 19:53:43 2019 +0100 + + Shortcut counts on exists queries (#39570) (#39660) + + `TopDocsCollectorContext` can already shortcut hit counts on `match_all` and `term` queries when there are no deletions. + This change adds this ability for `exists` queries if the index doesn't have deletions and fields are indexed. + + Closes #37475 + +commit af4918ebff75d5d438276d6a698bc8f3932c4a0a +Author: Nhat Nguyen +Date: Mon Mar 4 12:47:43 2019 -0500 + + Simplify AutoFollowCoordinator with GroupedListener (#39603) + + This change simplifies AutoFollowCoordinator by replacing a combination + of AtomicArray and CountDown with GroupedActionListener. + +commit 7b8ff2d7c574274b008f11d3b7f253e4f41b771b +Author: Jack Conradson +Date: Mon Mar 4 08:26:44 2019 -0800 + + Add tests for Painless casting from short and Short (#39587) + + This adds tests for casting from short and Short to other standard types in + Painless. This also corrects a few errors from byte and Byte cast tests. + +commit 98925e9a09ac0b3765a288cad5fec0e346c7dce4 +Author: Prabhakar S +Date: Mon Mar 4 23:04:09 2019 +0530 + + Fixing the custom object serialization bug in diffable utils. (#39544) + + While serializing custom objects, the length of the list is computed after + filtering out the unsupported objects but while writing objects the filter + is not applied thus resulting in writing unsupported objects which will fail + to deserialize by the receiever. Adding the condition to filter out unsupported + custom objects. + +commit 691b4e9709d8833071416d90837b535d327d27bd +Author: Martijn van Groningen +Date: Mon Mar 4 16:53:05 2019 +0100 + + mute test + +commit a9d74711fa08d35732d5ce9dafb269a6a3625b61 +Author: Armin Braun +Date: Mon Mar 4 16:50:50 2019 +0100 + + Reenable BwC Tests 7.x (#39650) + + * #39550 is merged so we can reenable these + +commit 801f13f2014d6f05aad585f669af214125cd8185 +Author: Nhat Nguyen +Date: Mon Mar 4 08:59:02 2019 -0500 + + Assert recovery done in testDoNotWaitForPendingSeqNo (#39595) + + Since #39006 we should be able to complete a peer-recovery without + waiting for pending indexing operations. Thus, the assertion in + testDoNotWaitForPendingSeqNo should be updated from false to true. + + Closes #39510 + +commit 15ab0e23e090ae894074debd77d3ebf5741b2b3a +Author: Nhat Nguyen +Date: Mon Mar 4 09:03:39 2019 -0500 + + Unmute testHistoryUUIDIsGenerated + + Fixed in #32270 + Relates to #31291 + +commit 936dbb00e31a5c238f355d2bb90beb83b7edbf4e +Author: Yannick Welsch +Date: Mon Mar 4 15:51:17 2019 +0100 + + Isolate Zen1 (#39470) + + Cherry-picks a few commits from #39466 to align 7.x with master branch. + +commit c72a7998f52f3442ef3f04be7b3e77193371b3b1 +Author: Marios Trivyzas +Date: Mon Mar 4 15:29:10 2019 +0100 + + SQL: Don't allow inexact fields for MIN/MAX (#39563) + + MIN/MAX on strings are supported and are implemented with + TopAggs FIRST/LAST respectively, but they cannot operate on + `text` fields without underlying `keyword` fields => inexact. + + Follows: #39427 + +commit 52ecf18dc43cc38cd4232f062378fc71c6b89217 +Author: Martijn Laarman +Date: Mon Mar 4 15:23:18 2019 +0100 + + Index on rollup.rollup_search.json is a list (#39097) (#39653) + + And not a string since it accepts comma separated list of indices. + + (cherry picked from commit cf34d50b3a983b5fc0c9c7aa279cecd4aa10e28b) + +commit c2a94aabbcc4bde9e83019250ee0e163d84b0fb0 +Author: Martijn Laarman +Date: Mon Mar 4 15:23:01 2019 +0100 + + ilm.explain_lifecycle documents human again (#39113) (#39648) + + This is already exposed as a `_common.json` global parameter. + + (cherry picked from commit e84050c0307bb5d5cea8eacc6b63b34248a41a01) + +commit 9788036857588a6513d3669b6aba58acfc82176e +Author: Martijn Laarman +Date: Mon Mar 4 15:22:44 2019 +0100 + + metric on watcher stats is a list not an enum (#39114) (#39645) + + `enum` is a single option from a known list of `options` + `list` is an array of unknown values + `flags` are multiple options from a list of known `options`. + + We don't support the `flags` type but a `list` with `options` acts as one. This is already the case for other API's taking metric such as `node.stats.json`. + + watcher.stats behaves the same as other API's as `metrics` and as such accepts the following `GET _xpack/watcher/stats/queued_watches,current_watches` + + (cherry picked from commit 4c00a025b8ac9b397b27c4ae2f799553d6499412) + +commit 7c69fd9e44eb21f3e13b41319e20100a4dab9587 +Author: Martijn Laarman +Date: Mon Mar 4 15:22:26 2019 +0100 + + parts documented as optional are actually required (#39122) (#39641) + + (cherry picked from commit e0f728b44ad49e28477767b3ee783a07ddf4bb0d) + +commit a58145f9e65d65533218b823754a3821bd372336 +Author: David Kyle +Date: Mon Mar 4 13:52:05 2019 +0000 + + [ML] Transition to typeless (mapping) APIs (#39573) + + ML has historically used doc as the single mapping type but reindex in 7.x + will change the mapping to _doc. Switching to the typeless APIs handles + case where the mapping type is either doc or _doc. This change removes + deprecated typed usages. + +commit 9ddaabba88aa1a3bc2041900bc76536e16d8e51e +Author: Luca Cavanna +Date: Mon Mar 4 09:52:23 2019 +0100 + + Remote private SearchHits.Total class (#39556) + + This is now possible as Lucene's `TotalHits` implements `equals`/`hashcode`, + all the other methods can be in-lined in `SearchHits` instead, no need for + a specific wrapper class. + +commit b8659fcb83f7f2db4dbb85428a0654ae31b7fdec +Author: Martijn van Groningen +Date: Mon Mar 4 13:26:43 2019 +0100 + + No need to extend from StatusToXContentObject, + if RestToXContentListener is used instead of RestStatusToXContentListener + +commit 547af21a12660ae9d471f87efe7da16c4c0a6666 +Author: Armin Braun +Date: Mon Mar 4 12:56:46 2019 +0100 + + Introduce Mapping ActionListener (#39538) (#39636) + + * Introduce Safer Chaining of Listeners + + * The motivation here is to make reasoning about chains of `ActionListener` a little easier, by providing a safe method for nesting `ActionListener` that guarantees that a response is never dropped. Also, it dries up the code a little by removing the need to repeat `listener::onFailure` and `listener.onResponse` over and over. + * Refactored a number of obvious/easy spots to use the new listener constructor + +commit c7a2910cc145442ac31617fc8414c72902c5fbd2 +Author: David Kyle +Date: Mon Mar 4 11:48:03 2019 +0000 + + [Ml-Dataframe] Register Data Frame named writables and xcontents (#39635) + + Register types in the Dataframe plugin + +commit fca6a2f006afd2cf7eda7ff89b0b678ad6722a7b +Author: Daniel Mitterdorfer +Date: Mon Mar 4 11:37:59 2019 +0100 + + Avoid deprecated API usage in TaskOperationFailure (#39303) (#39628) + + With this commit we remove usage of the deprecated method + `ExceptionsHelper#detailedMessage` in the class `TaskOperationFailure`. + + Relates #19069 + +commit dd68244841157a6d8961ce00fc021b8907b57c2e +Author: David Turner +Date: Mon Mar 4 10:26:07 2019 +0000 + + Wait for state recovery in testFreshestMasterElectedAfterFullClusterRestart (#39602) + + Zen1IT#testFreshestMasterElectedAfterFullClusterRestart fails sometimes because + we request the cluster state before state recovery has completed, and therefore + obtain the default value for the setting we're relying on. + + Confusingly, we were starting out by setting this setting to its default value, + so the test looked like it was failing because of a production bug. This commit + avoids this confusion in future by setting it to a non-default value at the + start of the test. + + Fixes #39586. + +commit 782f873165660755c2470b2bcaf185de91c8ef89 +Author: Adrien Grand +Date: Mon Mar 4 10:58:43 2019 +0100 + + Don't swallow exceptions in Store#close(). (#39035) (#39622) + + Store#close() swallows any `IOException`. + + Relates #39030 + +commit 934946a2325573983fb1a27dc68caa7f6937cfac +Author: Adrien Grand +Date: Mon Mar 4 10:58:29 2019 +0100 + + Don't swallow exception in ThreadPool.terminate. (#39038) (#39623) + + The use of `closeWhileHandlingException` means that any exception while trying + to close the threadpool is going to be swallowed. + + Relates #39030 + +commit 21540a5adacf66d46a539497d3e0cd73127b8fa7 +Author: Adrien Grand +Date: Mon Mar 4 10:58:12 2019 +0100 + + Enhancements to IndicesQueryCache. (#39099) (#39626) + + This commit adds the following: + - more tests to IndicesServiceCloseTests, one of them found a bug in the order + in which `IndicesQueryCache#onClose` and + `IndicesService.indicesRefCount#decRef` are called. + - made `IndicesQueryCache.stats2` a synchronized map. All writes to it are + already protected by the lock of the Lucene cache, but the final read from + an assertion in `IndicesQueryCache#close()` was not so this change should + avoid any potential visibility issues. + - human-readable `toString`s to make debugging easier. + + Relates #37117 + +commit 68bc178017a5b87a72601be76b760a988435df8f +Author: Armin Braun +Date: Mon Mar 4 10:41:52 2019 +0100 + + Disable Bwc Tests (#39551) + + * Disable Bwc Tests + * For #39550 + +commit 65732d707f7d23694c13490954ea9197f2c1a097 +Author: Armin Braun +Date: Mon Mar 4 10:32:37 2019 +0100 + + Add Support for S3 Intelligent Tiering (#39376) (#39620) + + * Add support for S3 intelligent tiering + * Closes #38836 + +commit 0f65390c29a9648c9288596bb17e2f763b6d53d9 +Author: Yannick Welsch +Date: Mon Mar 4 10:10:25 2019 +0100 + + Do not mutate engine during planning step (#39571) + + This cleans up the Engine implementation by separating the sequence number generation from the + planning step in the engine, to avoid for the planning step to have any side effects. This makes it + easier to see that every sequence number is properly accounted for. + +commit 0550ead176394f993aff2e0875cfc4ce3860f43b +Author: Martijn van Groningen +Date: Mon Mar 4 08:50:38 2019 +0100 + + Cleanup GrokProcessorGetAction class (#39567) + + * Removed request builder. From 7.0, request builders are no longer used. + * Use RestStatusToXContentListener instead of custom RestBuilderListener in the rest action. + * Changed a few public constructor's and constants' visibility from public to package protected. + (these are only used internally, so no need to for public visibility) + +commit b665d621927ad0bc6ae6409cf5bba9cde4dc2bba +Author: Adrien Grand +Date: Fri Mar 1 15:16:52 2019 +0100 + + Fix formatting of TESTING.asciidoc. + +commit 0f85182abe8ca542289b05f4467b5e9364a92194 +Author: Alpar Torok +Date: Mon Mar 4 09:12:12 2019 +0200 + + Testclusters: implement support to install plugins (#39116) + + * methods to run bin script + * Add support for specifying and installing plugins + * Add OS specific distirbution support + * Add test to verify plugin installed + * Remove use of Gradle internal OperatingSystem + +commit 813351fe261007e9147b61d8ff413b1e1df7aa15 +Author: Alpar Torok +Date: Mon Mar 4 08:37:43 2019 +0200 + + Un-mute and fix BuildExamplePluginsIT (#38899) + + * Un-mute and fix BuildExamplePluginsIT + + There doesn't seem to be anything wrong with the test iteself. + I think the failure were CI performance related, but while it was muted, + some failures managed to sneak in. + + Closes #38784 + + * PR review + +commit 9ec24bae806159e8fdb72c313b0f1e32fb09aa69 +Author: David Turner +Date: Sat Mar 2 13:56:33 2019 +0000 + + Mute testDoNotWaitForPendingSeqNo + + Relates #39510, #39595. + +commit 834a88abf9cf1d5fa5c2cc3caad91136bcc2cfdc +Author: Tim Vernum +Date: Mon Mar 4 12:57:51 2019 +1100 + + Mute failing test on FIPS JVM + + Relates: #39580 + Backport of: #39616 + +commit 085ff38122ad7d41f1ecc7aa8e35c97cb1bff6e5 +Author: David Roberts +Date: Sun Mar 3 18:33:27 2019 +0000 + + Mute DeleteExpiredDataIT.testDeleteExpiredData + + Due to https://github.com/elastic/elasticsearch/issues/39575 + +commit f5fb93afdff5925e53b9ec29d5541d8f66285ce2 +Author: David Turner +Date: Sat Mar 2 12:26:28 2019 +0000 + + Mute FullClusterRestartIT#testClosedIndices (#39600) + + Relates #39576 + +commit e038ccef134bf1e7df6a001c181e2e82cbe04c17 +Author: Costin Leau +Date: Sat Mar 2 10:29:48 2019 +0200 + + SQL: Fix merging of incompatible multi-fields (#39560) + + Fix bug in IndexResolver that caused conflicts in multi-field types to + be ignored up (causing the query to fail later on due to mapping + conflicts). + The issue was caused by the multi-field which forced the parent creation + before checking its validity across mappings + + Fix #39547 + + (cherry picked from commit 4e4fe289f90b9b5eae09072d54903701a3128696) + +commit dfe81b260ed5ca81862552732160386050867765 +Author: Costin Leau +Date: Fri Mar 1 23:06:16 2019 +0200 + + SQL: Enable accurate hit tracking on demand (#39527) + + Queries that require counting of all hits (COUNT(*) on implicit + group by), now enable accurate hit tracking. + + Fix #37971 + + (cherry picked from commit 265b637cf6df08986a890b8b5daf012c2b0c1699) + +commit f1a7166708a377c5ae170dce6d999b66a6e28704 +Author: Lisa Cawley +Date: Fri Mar 1 10:29:59 2019 -0800 + + [DOCS] Adds link to list of built-in users (#39529) + +commit 2a20e8e3fe2bf384062adabc0643d20d6ebec4eb +Author: James Baiera +Date: Fri Mar 1 13:27:57 2019 -0500 + + Remove missing variable from error message (#39321) + + When test clusters are stood up, one of the steps in the wait task is to wait for + ports files to appear. An exception throw was added if this were to time out + instead of failing with no information, but the exception text uses a missing + variable which further obfuscates the problem. + + Backports #39321 + +commit d0e65a45a23f35c37ed0fae6381efd256684e504 +Author: Mayya Sharipova +Date: Thu Feb 28 15:41:30 2019 -0500 + + Add debug log for flush for IndicesRequestCacheIT (#39475) + + Add debug log when index is flushed to investigate a failure + in IndicesRequestCacheIT + + "DEBUG" level is used as "TRACE" produces too much output irrelevant for this + issue + + Relates to #32827 + +commit 88438320397310b7294c2e497686cb982d3dfad6 +Author: Dimitris Athanasiou +Date: Fri Mar 1 17:17:26 2019 +0200 + + [ML] Shave off DeleteExpiredDataIT runtime (#39557) + + This commit parallelizes some parts of the test + and its remove an unnecessary refresh call. + On my local machine it shaves off about 15 seconds + for a test execution time of ~64s (down from ~80s). + This test is still slow but progress over perfection. + + Relates #37339 + +commit 0c6b7cfb77ff84c78ca97bc4905c81ff1ee64b43 +Author: Tanguy Leroux +Date: Fri Mar 1 17:54:19 2019 +0100 + + Revert "Support concurrent refresh of refresh tokens (#39559)" + + This reverts commit e2599214e0e4635c066f420277c30dce76278168. + +commit 39a401b82742ff9d6497115728b3628b46a189ae +Author: Jack Conradson +Date: Thu Feb 28 15:18:28 2019 -0800 + + Remove non-existent variable from Painless context docs (#39523) + +commit 687a66b5807879ab7663914d0acd21979cf1e80b +Author: Jack Conradson +Date: Wed Feb 27 14:57:58 2019 -0800 + + Add byte and Byte to Painless standard cast tests (#39415) + +commit 976f988358a2875575c6cbd148f8dfd1c5f867d1 +Author: Adrien Grand +Date: Fri Mar 1 15:13:39 2019 +0100 + + Add guidance for writing tests. (#39318) + +commit e2599214e0e4635c066f420277c30dce76278168 +Author: Ioannis Kakavas +Date: Fri Mar 1 16:00:07 2019 +0200 + + Support concurrent refresh of refresh tokens (#39559) + + This is a backport of #38382 + + This change adds supports for the concurrent refresh of access + tokens as described in #36872 + In short it allows subsequent client requests to refresh the same token that + come within a predefined window of 60 seconds to be handled as duplicates + of the original one and thus receive the same response with the same newly + issued access token and refresh token. + In order to support that, two new fields are added in the token document. One + contains the instant (in epoqueMillis) when a given refresh token is refreshed + and one that contains a pointer to the token document that stores the new + refresh token and access token that was created by the original refresh. + A side effect of this change, that was however also a intended enhancement + for the token service, is that we needed to stop encrypting the string + representation of the UserToken while serializing. ( It was necessary as we + correctly used a new IV for every time we encrypted a token in serialization, so + subsequent serializations of the same exact UserToken would produce + different access token strings) + + This change also handles the serialization/deserialization BWC logic: + + - In mixed clusters we keep creating tokens in the old format and + consume only old format tokens + - In upgraded clusters, we start creating tokens in the new format but + still remain able to consume old format tokens (that could have been + created during the rolling upgrade and are still valid) + + Resolves #36872 + + Co-authored-by: Jay Modi jaymode@users.noreply.github.com + +commit 29e3c18713f96712d095ba23f2f4b23b38ec6b49 +Author: Luca Cavanna +Date: Fri Mar 1 14:54:48 2019 +0100 + + Mute failing IndexShardIT#testPendingRefreshWithIntervalChange + + Relates to #39565 + +commit e005eeb0b389ef6c6ba8c246eb8021d32e510461 +Author: Tanguy Leroux +Date: Fri Mar 1 14:48:26 2019 +0100 + + Backport support for replicating closed indices to 7.x (#39506)(#39499) + + Backport support for replicating closed indices (#39499) + + Before this change, closed indexes were simply not replicated. It was therefore + possible to close an index and then decommission a data node without knowing + that this data node contained shards of the closed index, potentially leading to + data loss. Shards of closed indices were not completely taken into account when + balancing the shards within the cluster, or automatically replicated through shard + copies, and they were not easily movable from node A to node B using APIs like + Cluster Reroute without being fully reopened and closed again. + + This commit changes the logic executed when closing an index, so that its shards + are not just removed and forgotten but are instead reinitialized and reallocated on + data nodes using an engine implementation which does not allow searching or + indexing, which has a low memory overhead (compared with searchable/indexable + opened shards) and which allows shards to be recovered from peer or promoted + as primaries when needed. + + This new closing logic is built on top of the new Close Index API introduced in + 6.7.0 (#37359). Some pre-closing sanity checks are executed on the shards before + closing them, and closing an index on a 8.0 cluster will reinitialize the index shards + and therefore impact the cluster health. + + Some APIs have been adapted to make them work with closed indices: + - Cluster Health API + - Cluster Reroute API + - Cluster Allocation Explain API + - Recovery API + - Cat Indices + - Cat Shards + - Cat Health + - Cat Recovery + + This commit contains all the following changes (most recent first): + * c6c42a1 Adapt NoOpEngineTests after #39006 + * 3f9993d Wait for shards to be active after closing indices (#38854) + * 5e7a428 Adapt the Cluster Health API to closed indices (#39364) + * 3e61939 Adapt CloseFollowerIndexIT for replicated closed indices (#38767) + * 71f5c34 Recover closed indices after a full cluster restart (#39249) + * 4db7fd9 Adapt the Recovery API for closed indices (#38421) + * 4fd1bb2 Adapt more tests suites to closed indices (#39186) + * 0519016 Add replica to primary promotion test for closed indices (#39110) + * b756f6c Test the Cluster Shard Allocation Explain API with closed indices (#38631) + * c484c66 Remove index routing table of closed indices in mixed versions clusters (#38955) + * 00f1828 Mute CloseFollowerIndexIT.testCloseAndReopenFollowerIndex() + * e845b0a Do not schedule Refresh/Translog/GlobalCheckpoint tasks for closed indices (#38329) + * cf9a015 Adapt testIndexCanChangeCustomDataPath for replicated closed indices (#38327) + * b9becdd Adapt testPendingTasks() for replicated closed indices (#38326) + * 02cc730 Allow shards of closed indices to be replicated as regular shards (#38024) + * e53a9be Fix compilation error in IndexShardIT after merge with master + * cae4155 Relax NoOpEngine constraints (#37413) + * 54d110b [RCI] Adapt NoOpEngine to latest FrozenEngine changes + * c63fd69 [RCI] Add NoOpEngine for closed indices (#33903) + + Relates to #33888 + +commit 06d0e0efad00d6ec7bf9e0103881692bddb3b4a1 +Author: Andrei Stefan +Date: Fri Mar 1 15:23:38 2019 +0200 + + Removed custom naming for DISTINCT COUNT (#39537) + + (cherry picked from commit 9412a2ee01a60dd6449bbced1273ec0b37b65589) + +commit ba44f28340704ca9e696b4eab6aba6b90199ca66 +Author: Andrei Stefan +Date: Fri Mar 1 15:23:15 2019 +0200 + + SQL: ignore UNSUPPORTED fields for JDBC and ODBC modes in 'SYS COLUMNS' (#39518) + + * SYS COLUMNS will skip UNSUPPORTED field types in ODBC and JDBC, as well. + NESTED and OBJECT types were already skipped in ODBC mode, now they are + skipped in JDBC mode, as well. + + (cherry picked from commit 9e0df64b2d36c9069dfa506570468f0522c86417) + +commit 894ecb244dd1cfa5eb190d58b8673f5a43618983 +Author: David Kyle +Date: Fri Mar 1 10:45:36 2019 +0000 + + [ML-Dataframe] Move dataframe actions into core (#39548) + +commit 9fb2f670dc9c946fb0c0392d30442a52abd3d720 +Author: Marios Trivyzas +Date: Fri Mar 1 00:10:30 2019 +0100 + + SQL: Enhance checks for inexact fields (#39427) + + For functions: move checks for `text` fields without underlying `keyword` + fields or with many of them (ambiguity) to the type resolution stage. + + For Order By/Group By: move checks to the `Verifier` to catch early + before `QueryTranslator` or execution. + + Closes: #38501 + Fixes: #35203 + +commit 1a50af7dd4a8696a26e218f4ca4433dcdc5da541 +Author: Yannick Welsch +Date: Fri Mar 1 09:16:53 2019 +0100 + + Do not close bad indices on startup (#39500) + + With #17187, we verified IndexService creation during initial state recovery on the master and if the + recovery failed the index was imported as closed, not allocating any shards. This was mainly done to + prevent endless allocation loops and full log files on data-nodes when the indexmetadata contained + broken settings / analyzers. Zen2 loads the cluster state eagerly, and this check currently runs on all + nodes (not only the elected master), which can significantly slow down startup on data nodes. + Furthermore, with replicated closed indices (#33888) on the horizon, importing the index as closed + will no longer not allocate any shards. Fortunately, the original issue for endless allocation loops is + no longer a problem due to #18467, where we limit the retries of failed allocations. The solution here + is therefore to just undo #17187, as it's no longer necessary, and covered by #18467, which will solve + the issue for Zen2 and replicated closed indices as well. + +commit b9b46fdec6aa6217b89e91031845d9e3f4fb8bb5 +Author: Tal Levy +Date: Thu Feb 28 21:02:32 2019 -0800 + + fix UpdateSettingsRequestStreamableTests.mutateInstance (#39386) (#39477) + + Mutations of the timeout values were using string-representations. + + This resulted in very rare cases where the original timeout value was + represented as something like "0ms" and the new random time-value generated + was "0s". Although their string representations differ, their underlying + TimeValue does not. This resulted in `-Dtests.seed=7F4C034C43C22B1B` to + fail. + +commit 609118c2299311573e6cad52f5a32768ce1b03f6 +Author: Mark Tozzi +Date: Thu Feb 28 15:40:09 2019 -0500 + + Override and mute InternalAutoDateHistogramTests#testReduceRandom() (#39536) + + pending resolution of #39497 + +commit 1124624e87c09617fd5be8da08e1396233ea47a7 +Author: Ryan Ernst +Date: Thu Feb 28 12:35:23 2019 -0800 + + Obsolete pre 7.0 noarch package in rpm (#39472) + + This commit makes the rpm metadata indicate the pre 7.0 noarch packages + are obsoleted by this package. This fixes an issue where upgrading with + yum would cause an error thinking there was nothing to upgrade. + + closes #39414 + +commit 8ced21db880f439a5c915ee18d90f30103004e58 +Author: Shajahan Palayil +Date: Thu Feb 28 20:08:39 2019 +0100 + + [DOCS] Corrected API path for /_security/api_key (#39521) + +commit dae48ba2628aae7cc0841dad8783df79342cc846 +Author: Lee Hinman +Date: Thu Feb 28 10:50:24 2019 -0700 + + Add details about what acquired the shard lock last (#38807) + + This adds a `details` parameter to shard locking in `NodeEnvironment`. This is + intended to be used for diagnosing issues such as + + ``` + 1> [2019-02-11T14:34:19,262][INFO ][o.e.c.m.MetaDataDeleteIndexService] [node_s0] [.tasks/oSYOG0-9SHOx_pfAoiSExQ] deleting index + 1> [2019-02-11T14:34:19,279][WARN ][o.e.i.IndicesService ] [node_s0] [.tasks/oSYOG0-9SHOx_pfAoiSExQ] failed to delete index + 1> org.elasticsearch.env.ShardLockObtainFailedException: [.tasks][0]: obtaining shard lock timed out after 0ms + 1> at org.elasticsearch.env.NodeEnvironment$InternalShardLock.acquire(NodeEnvironment.java:736) ~[main/:?] + 1> at org.elasticsearch.env.NodeEnvironment.shardLock(NodeEnvironment.java:655) ~[main/:?] + 1> at org.elasticsearch.env.NodeEnvironment.lockAllForIndex(NodeEnvironment.java:601) ~[main/:?] + 1> at org.elasticsearch.env.NodeEnvironment.deleteIndexDirectorySafe(NodeEnvironment.java:554) ~[main/:?] + ``` + + In the hope that we will be able to determine why the shard is still locked. + + Relates to #30290 as well as some other CI failures + +commit e564c4d8adb1fc355cc1e1d23b8b5d059df19ab6 +Author: Armin Braun +Date: Thu Feb 28 18:23:01 2019 +0100 + + Add Package Level JavaDoc on Snapshots (#38108) (#39514) + + * Add Package Level JavaDoc on Snapshots + +commit 8a19d981db6f82339a813f826d63126a0794e3d9 +Author: Albert Zaharovits +Date: Thu Feb 28 14:40:16 2019 +0200 + + Integ test snapshot and restore for native realm (#39123) + + This commit adds a simple integ test that exercises the flow: + * snapshot .security + * delete .security + * restore .security + + , checking that the Native Realm works as expected. + + Relates #34454 + +commit 30e5c11cc2a8158d557aff38492e3f8228c3c6ea +Author: Hendrik Muhs +Date: Thu Feb 28 13:00:37 2019 +0100 + + [ML-DataFrame] Dataframe REST cleanups (#39451) (#39503) + + fix a couple of odd behaviors of data frame transforms REST API's: + + - check if id from body and id from URL match if both are specified + - do not allow a body for delete + - allow get and stats without specifying an id + +commit 5c96b90ed57b1249abb57349af47c0284e58c7d1 +Author: Simon Willnauer +Date: Thu Feb 28 11:57:20 2019 +0100 + + Never block on scheduled refresh if a refresh is running (#39462) + + Today we block on the ReferenceManager in the case of a scheduled refresh. + Yet if there is a refresh happening concurrently we might block and create + very smallish segments. Instead we should just move on to the next shard + and free up the refresh thread instead. + +commit d3d7d9bb9d41c7606a4d575dc88d8f3bffdec676 +Author: Armin Braun +Date: Thu Feb 28 10:33:05 2019 +0100 + + Remove Dead Code + Duplication in o.e.c.routing (#36678) (#39493) + + * Removed obviously unused fields+methods + * Inlined public methods that only had one caller + * Simplified `Optional` chain + * Simplified some obviously redundant conditions + +commit 8122650a55c3b6d1830f2f363ca9258ed940662d +Author: Dimitris Athanasiou +Date: Thu Feb 28 11:08:27 2019 +0200 + + [ML] Add integration test for interim results after advancing bucket (#39447) + + This is an integration test that captures the issue described in + elastic/ml-cpp#324 + +commit 90ab4a6f6e23505fe4fd19d8257e85b00ddd91a4 +Author: Armin Braun +Date: Thu Feb 28 08:01:52 2019 +0100 + + Stabilize RareClusterState (#38671) (#39468) + + * Use actual master node, not just a master elligible node when trying to cancel publication. This only works on the master and for unlucky seeds we never try the master within the 10s that the busy assert runs. + * Closes #36813 + +commit 2ce9457c8f732b0a10ef2bbe3d8da2da1bcd2a8a +Author: Ioannis Kakavas +Date: Wed Feb 27 16:15:21 2019 +0200 + + Mute Bulk indexing of monitoring data (#39448) + + Relates: #30101 + +commit f538b30af9e61cf1c2929dcae6850b1c280cae3d +Author: Tal Levy +Date: Wed Feb 27 15:56:33 2019 -0800 + + ensure no initializing shards during cluster cleanup (#39283) (#39480) + + there are testing situations where newly created indices + are being wiped before they are fully initialized. This results + in an edge-case in the shard-locking strategy where an index + cannot be deleted. + + This should fix that + +commit 8b26f599585436f4d077c9e73f1ceb065f697d34 +Author: Lisa Cawley +Date: Wed Feb 27 15:33:50 2019 -0800 + + [DOCS] Removes problematic footer from Watcher docs (#39474) + +commit 4dd274b51d348a5b1755cfcce3067df5032b5f3f +Author: Tanguy Leroux +Date: Wed Feb 27 20:36:59 2019 +0100 + + Unmute CoordinatorTests.testDiscoveryUsesNodesFromLastClusterState() (#39452) + + This commit unmutes the test and comments out the + offending call to linearizabilityChecker.isLinearizable() as suggested + in #39437 + +commit ad8228aec9e11451822bc0dc1411b61df9cc764b +Author: Lee Hinman +Date: Wed Feb 27 11:11:19 2019 -0700 + + Use non-ILM template setting up watch history template & ILM disabled (#39420) + + Backport of #39325 + + When ILM is disabled and Watcher is setting up the templates and policies for + the watch history indices, it will now use a template that does not have the + `index.lifecycle.name` setting, so that indices are not created with the + setting. + + This also adds tests for the behavior, and changes the cluster state used in + these tests to be real instead of mocked. + + Resolves #38805 + +commit 9c8c158f21592596c8371e5b1300d8c86bf0ff61 +Author: Lisa Cawley +Date: Wed Feb 27 09:43:21 2019 -0800 + + [DOCS] Fix inline callout in Watcher documentation (#39423) + +commit 983b5d1c0eb1b1feb9e7c36efc9d795631041786 +Author: Tanguy Leroux +Date: Wed Feb 27 17:54:54 2019 +0100 + + Mute SpecificMasterNodesIT.testElectOnlyBetweenMasterNodes() + + Tracked in #38331 + +commit 54ced2949ba9e4786943f82a69e49b3baf0b7f39 +Author: Alan Woodward +Date: Wed Feb 27 16:51:15 2019 +0000 + + Re-enable BWC (#39460) + + Follow up to #39444 + +commit 6d72d45e339f3125e347605850f973d07cc5da5e +Author: Jason Tedor +Date: Wed Feb 27 11:39:49 2019 -0500 + + Use https to obtain Lucene snapshots (#39458) + + This commit changes the protocol used to download Lucene snapshots. + +commit dedbe60e0a26412766bce5cd78ace38766ffe2eb +Author: Lisa Cawley +Date: Wed Feb 27 08:15:01 2019 -0800 + + [DOCS] Fixes table and code block separators in Watcher documentation (#39426) + +commit 995144b197d4b50eae5f896775cc3771fd4da8e4 +Author: Jay Modi +Date: Wed Feb 27 08:17:44 2019 -0700 + + Fix SSLConfigurationReloaderTests failure tests (#39408) + + This change fixes the tests that expect the reload of a + SSLConfiguration to fail. The tests relied on an incorrect assumption + that the reloader only called reload on for an SSLConfiguration if the + key and trust managers were successfully reloaded, but that is not the + case. This change removes the fail call with a wrapped call to the + original method and captures the exception and counts down a latch to + make these tests consistently tested. + + Closes #39260 + +commit 2ccba188095070b44cf6e021556cfe0870d54956 +Author: Daniel Mitterdorfer +Date: Wed Feb 27 17:03:50 2019 +0100 + + Correct name of basic_date_time_no_millis (#39367) (#39454) + + With this commit we correct the name of the Java time based formatter + for `basic_date_time_no_millis`. + +commit e6c2dae250c94f73bac88ecf83b79f466c98ec3d +Author: Lisa Cawley +Date: Wed Feb 27 07:34:06 2019 -0800 + + [DOCS] Fix image warnings in CCR documentation (#39430) + +commit 71b84941811e9a20fe9150100661efe276ba2696 +Author: Alan Woodward +Date: Wed Feb 27 14:36:08 2019 +0000 + + Upgrade to lucene 8.0.0-snapshot-ff9509a8df (#39444) + + Backport of #39350 + + Contains the following: + + * LUCENE-8635: Move terms dictionary off-heap for non-primary-key fields in `MMapDirectory` + * LUCENE-8292: `TermsEnum` is fully abstract + * LUCENE-8679: Return WITHIN in `EdgeTree#relateTriangle` only when polygon and triangle share one edge + * LUCENE-8676: Nori tokenizer deals correctly with large buffers + * LUCENE-8697: `GraphTokenStreamFiniteStrings` better handles side paths with gaps + * LUCENE-8664: Add `equals` and `hashCode` to `TotalHits` + * LUCENE-8660: `TopDocsCollector` returns accurate hit counts if the total equals the threshold + * LUCENE-8654: `Polygon2D#relateTriangle` fix for when the polygon is inside the triangle + * LUCENE-8645: `Intervals#fixField` can merge intervals from different fields + * LUCENE-8585: Create jump-tables for DocValues at index time + +commit f675b33d50d1ea99b3284e2bfab3b7784919e37b +Author: Armin Braun +Date: Wed Feb 27 15:22:17 2019 +0100 + + Increase Timeout in UnicastZenPingTests (#38893) (#39449) + + * Just like #37268 removing another 1s timeout, those are dangerous since they're easily exceeded by an untimely gc pause + * Closes #26701 + +commit a2c07b50116d4f7f7e69b4393356a0168131f1ed +Author: Marios Trivyzas +Date: Wed Feb 27 14:45:39 2019 +0100 + + SQL: Use underlying exact field for LIKE/RLIKE (#39443) + + Previously, if a text field had an underlying keyword field + the latter was not used instead of the text leading to wrong + results returned by queries filtering with LIKE/RLIKE. + + Fixes: #39442 + +commit 55e98f08d89ac30c2d17a143dd8811261ce9be62 +Author: Jason Tedor +Date: Wed Feb 27 08:10:02 2019 -0500 + + Provide a clearer error message on keystore add (#39327) + + When trying to add a setting to the keystore with an upper case name, we + reject with an unclear error message. This commit makes that error + message much clearer. + +commit 6c5bf3ac13346ec83bcd8c295fdaa1ad59ffe206 +Author: Jason Tedor +Date: Wed Feb 27 08:08:06 2019 -0500 + + Remove outdated DNS caching docs from HTTP exporter (#39394) + + These docs are out of date, now that we override the infinite DNS cache + within Elasticsearch. This commit completely removes this content, as + specific guidance is no longer needed here. + +commit 842940785a2792ceac3a5f9cb2d0c76776a03643 +Author: Jason Tedor +Date: Wed Feb 27 08:04:19 2019 -0500 + + Conditionally build BWC projects in parallel (#39396) + + This commit sets the BWC projects to build in parallel if Gradle was + invoked with parallal project execution enabled. This substantially + speeds up the time of building the BWC projects since there are many + dependent projects needed to build a BWC version. + +commit 27485871b805c41c225f89edefe185efaf249b71 +Author: Armin Braun +Date: Wed Feb 27 13:39:25 2019 +0100 + + Don't Ping on Handshake Connection (#39076) (#39446) + + * Don't Ping on Handshake Connection + + * It does not make sense to run pings on the handshake connection + * Set the ping interval to `-1` to deactivate pings on it + +commit 6912e27ee0fb68881088d1b22ca09055567384af +Author: Tanguy Leroux +Date: Wed Feb 27 13:13:22 2019 +0100 + + Mute MinimumMasterNodesIT.testThreeNodesNoMasterBlock() + + Tracked in #39172 + +commit da9190be0a41c294296dfabafd32a0b589ed82a9 +Author: Armin Braun +Date: Wed Feb 27 11:28:30 2019 +0100 + + Add Checks for Closed Channel in Selector Loop (#39096) (#39439) + + * A few warnings could be observed in test logs about `NoSuchElementException` being thrown in `InboundChannelBuffer#sliceBuffersTo`. + These were the result of calls to this method after the relevant channel and hence the buffer was closed already as a result of a failed IO operation. + * Fixed by adding the necessary guard statements to break out in these cases. I don't think there is a need here to do any additional error handling since `eventHandler.postHandling(channelContext);` at the end of the `processKey` + call in the main selection loop handles closing channels and invoking callbacks for writes that failed to go through already. + +commit 1d0097b5e83090072de5f8b62e36c2789d167722 +Author: Mehran Koushkebaghi +Date: Wed Feb 27 09:18:03 2019 +0000 + + [ML] Refactoring scheduled event to store instant instead of zoned time zone (#39380) + + The ScheduledEvent class has never preserved the time + zone so it makes more sense for it to store the start and + end time using Instant rather than ZonedDateTime. + + Closes #38620 + +commit a427a283188204925cc0a3f1db7d4b5e2b6f8352 +Author: Martijn van Groningen +Date: Wed Feb 27 09:20:41 2019 +0100 + + Unmuted testCannotFollowLeaderInUpgradedCluster test. + + Relates to #39355 + +commit 41668f772358473a4742139db41c6167df9dca0f +Author: David Turner +Date: Wed Feb 27 08:29:39 2019 +0000 + + Move PeerFinder's logger to the expected package (#39412) + + Today the abstract `org.elasticsearch.discovery.PeerFinder` uses the logger of + its implementation, which in production is in `o.e.cluster.coordination`. This + turns out to be confusing and unhelpful, so with this change we move to using + the logger that belongs to `PeerFinder`. + +commit 28b771f5db0d4a14e602f006fadfebc1c0c29d5f +Author: Armin Braun +Date: Wed Feb 27 09:38:47 2019 +0100 + + Remove Dead Code Test Infrastructure (#39192) (#39436) + + * Just removing some obviously unused things + +commit 542e2c55f6849c973b502e3669c0843325ed732a +Author: Andrei Stefan +Date: Wed Feb 27 09:48:10 2019 +0200 + + SQL: change the default precision for CURRENT_TIMESTAMP function (#39391) + + (cherry picked from commit dbb93310b083226c96e4bde3eef0079eb01cbca9) + +commit 4deb69e9e4bf2b85ae0bb5b138762a92c8e052c8 +Author: Andrei Stefan +Date: Wed Feb 27 09:24:52 2019 +0200 + + SQL: introduce the columnar option for REST requests (#39287) + + * Add "columnar" option for REST requests (but be lenient for non-"plain" + modes) for json, yaml, smile and cbor formats. + * Updated documentation + + (cherry picked from commit 5b7e0de237fb514d14a61a347bc669d4b4adbe56) + +commit d955375e9c3db149a5b3f3939c8b698af1a14b8b +Author: Alpar Torok +Date: Wed Feb 27 09:16:58 2019 +0200 + + Don't do BWC packaing tests if disabled (#39397) + + Packaging tests did not honor bwc tests being off. + This was also the reason for which we were building the BWC versinons + even if the tests are off, so this closes #35347. + +commit d16edf04625cf52d3ff89ca125bbe07797df35fa +Author: Andrei Stefan +Date: Wed Feb 27 07:30:41 2019 +0200 + + Randomize the timezone for equals and hashcode tests (#39353) + +commit f24dae302d80206e12e3aa6c121bc458b1fa740f +Author: Tim Brooks +Date: Tue Feb 26 18:54:38 2019 -0700 + + Make security tests transport agnostic (#39411) + + Currently there are two security tests that specifically target the + netty security transport. This PR moves the client authentication tests + into `AbstractSimpleSecurityTransportTestCase` so that the nio transport + will also be tested. + + Additionally the work to build transport configurations is moved out of + the netty transport and tested independently. + +commit 30687cbe7fc65111d29ca8f4e9fb828f2c32724e +Author: Tim Vernum +Date: Wed Feb 27 12:49:44 2019 +1100 + + Switch internal security index to ".security-7" (#39422) + + This changes the name of the internal security index to ".security-7", + but supports indices that were upgraded from earlier versions and use + the ".security-6" name. + + In all cases, both ".security-6" and ".security-7" are considered to + be restricted index names regardless of which name is actually in use + on the cluster. + + Backport of: #39337 + +commit 0c7310936be529ad99496d32a08de975da126379 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Feb 27 12:40:15 2019 +1100 + + Fixed required fields and paths list (#39358) (#39428) + + Some small fix for the `x-pack` rest api spec. + + * In both `security.enable_user.json` and `security.disable_user.json` + the `username` parameter was `false` instead of `true` + (the documentation is already correct). + * In `security.get_privileges.json` there were missing all the + possible paths since the path parameters are not required. + This fix aligns the document with the rest of the spec, + where all the possible combinations are listed. + +commit f4c5abe4d4e0f51b8032c758a11f1d4eb0570786 +Author: Gordon Brown +Date: Tue Feb 26 16:58:30 2019 -0700 + + Handle failure to release retention leases in ILM (#39281) (#39417) + + It is possible that the Unfollow API may fail to release shard history + retention leases when unfollowing, so this needs to be handled by the + ILM Unfollow action. There's nothing much that can be done automatically + about it from the follower side, so this change makes the ILM unfollow + action simply ignore those failures. + +commit a9e86bc941776083ca36525fdecd4b1944d1f199 +Author: Nhat Nguyen +Date: Tue Feb 26 15:51:11 2019 -0500 + + Adjust testWaitForPendingSeqNo (#39404) + + Since #39006, we should either remove `testWaitForPendingSeqNo` + or adjust it not to wait for the pending operations. This change picks + the latter. + + Relates #39006 + +commit 4ca514f18c4982a2f6cbafab78948fcbfd619dcb +Author: Mayya Sharipova +Date: Tue Feb 26 11:53:38 2019 -0500 + + Fix testCacheWithFilteredAlias failure (#39401) + + Move refresh after Forcemerge + + Relates to #32827 + +commit 2619f48e4d740d1a39484f14244c2756e1be0c8a +Author: Luca Cavanna +Date: Tue Feb 26 16:30:22 2019 +0100 + + Rename SearchRequest#withLocalReduction (#39108) + + `withLocalReduction` is confusing as `local` effectively means "local + to the remote clusters" rather than "local the coordinating node" where + the method is executed. I propose we rename the method to + `crossClusterSearch` which better resembles what the static method is + used for. + +commit 2f7206ada735ab1a663dd807bd75ca255c57b38e +Author: Neeraj Jain +Date: Tue Feb 26 20:56:21 2019 +0530 + + Use pkill to shutdown elasticsearch using pid file (#39135) + + While running these commands from alias, facing issues using kill `cat pid`, In some situations, the more compact: + ``` + pkill -F /var/run/myProcess.pid + ``` + is the way to go. + +commit c09773a76ec015cc87027db6098f0c571fb5c710 +Author: Luca Cavanna +Date: Tue Feb 26 11:41:22 2019 +0100 + + Completion suggestions to be reduced once instead of twice (#39255) + + We have been calling `reduce` against completion suggestions twice, once + in `SearchPhaseController#reducedQueryPhase` where all suggestions get + reduced, and once more in `SearchPhaseController#sortDocs` where we + add the top completion suggestions to the `TopDocs` so their docs can + be fetched. There is no need to do reduction twice. All suggestions can + be reduced in one call, then we can filter the result and pass only the + already reduced completion suggestions over to `sortDocs`. The small + important detail is that `shardIndex`, which is currently used only + to fetch suggestions hits, needs to be set before the first reduction, + hence outside of `sortDocs` where we have been doing it until now. + +commit 24e478c58e93438f29cb4309147dcae6a2c0d0a5 +Author: Martijn van Groningen +Date: Tue Feb 26 10:39:29 2019 +0100 + + Fix test, more than one node may be connected. + + Relates to #37681 + +commit f7cba82c775fa0e246e7ed28ede76e9421c8605c +Author: David Kyle +Date: Tue Feb 26 08:51:59 2019 +0000 + + [ML] Reenable ml rolling upgrade tests (#39290) + +commit 7f999c43b353675424498af847cfe66d4dc938cd +Author: Ioannis Kakavas +Date: Tue Feb 26 10:42:36 2019 +0200 + + [BACKPORT-7.x] Fix TokenBackwardsCompatibility tests (#39294) + + + This change is a backport of #39252 + + - Fixes TokenBackwardsCompatibilityIT: Existing tests seemed to made + the assumption that in the oneThirdUpgraded stage the master node + will be on the old version and in the twoThirdsUpgraded stage, the + master node will be one of the upgraded ones. However, there is no + guarantee that the master node in any of the states will or will + not be one of the upgraded ones. + This class now tests: + - That we can generate and consume tokens before we start the + rolling upgrade. + - That we can consume tokens generated in the old cluster during + all the stages of the rolling upgrade. + - That while on a mixed cluster, when/if the master node is + upgraded, we can generate, consume and refresh a token + - That after the rolling upgrade, we can consume a token + generated in an old cluster and can invalidate it so that it + can't be used any more. + - Ensures that during the rolling upgrade, the upgraded nodes have + the same configuration as the old nodes. Specifically that the + file realm we use is explicitly named `file1`. This is needed + because while attempting to refresh a token in a mixed cluster + we might create a token hitting an old node and attempt to refresh + it hitting a new node. If the file realm name is not the same, the + refresh will be seen as being made by a "different" client, and + will, thus, fail. + - Renames the Authentication variable we check while refreshing a + token to be clientAuth in order to make the code more readable. + + Some of the above were possibly causing the flakiness of #37379 + +commit b159cc51c051b1d2a845d9d347290c26c61aa522 +Author: Martijn van Groningen +Date: Tue Feb 26 09:05:49 2019 +0100 + + Ensure remote connection established and + clean remote connection prior to leader cluster restart + + Relates to #37681 + +commit d42f422258446c251670f37685086eb6555ad36a +Author: Yannick Welsch +Date: Tue Feb 26 16:19:19 2019 +0900 + + Add linearizability checker for coordination layer (#36943) + + Checks that the core coordination algorithm implemented as part of Zen2 (#32006) supports + linearizable semantics. This commit adds a linearizability checker based on the Wing and Gong + graph search algorithm with support for compositional checking and activates these checks for all + CoordinatorTests. + +commit b40628f6a6051b8f40e897b521512207a8dbbe6f +Author: Alpar Torok +Date: Tue Feb 26 09:10:23 2019 +0200 + + Don't fail bwc check if these are disabled (#38919) + +commit 575eed8582165b32c0b175c1c17d25bd1a53699a +Author: Nhat Nguyen +Date: Mon Feb 25 15:31:48 2019 -0500 + + Bubble up exception when processing NoOp (#39338) + + Today we do not bubble up exceptions when processing NoOps but always + treat them as document-level failures. This incorrect treatment causes + the assert_no_failure being tripped in peer-recovery if IndexWriter was + closed exceptionally before. + + Closes #38898 + +commit e9dda758347f824847cd3182c6c47b395293aed3 +Author: Nhat Nguyen +Date: Mon Feb 25 14:56:58 2019 -0500 + + Enable soft-deletes by default for 7.0+ indices (#38929) + + Today when users upgrade to 7.0, existing indices will automatically + switch to soft-deletes without an opt-out option. With this change, + we only enable soft-deletes by default for new indices. + + Relates #36141 + +commit a6c0166d68c04751a62f1ecd8361510ecd00cf84 +Author: Jason Tedor +Date: Mon Feb 25 17:07:55 2019 -0500 + + Renew retention leases while following (#39335) + + This commit is the final piece of the integration of CCR with retention + leases. Namely, we periodically renew retention leases and advance the + retaining sequence number while following. + +commit 7b8178c839a17c74116113e733cbfec5879d0a27 +Author: Lee Hinman +Date: Mon Feb 25 15:08:46 2019 -0700 + + Remove Hipchat support from Watcher (#39374) + + * Remove Hipchat support from Watcher (#39199) + + Hipchat has been shut down and has previously been deprecated in + Watcher (#39160), therefore we should remove support for these actions. + + * Add migrate note + +commit d5046b1c257007ecf9145c822bca16517ba67721 +Author: Igor Motov +Date: Mon Feb 25 12:56:19 2019 -0500 + + [CI] Fixes testQueryRandomGeoCollection failure again (#39275) + + Moves the check for tiny polygons earlier in the test. It turned out + that polygons can be so tiny that we cannot even figure out their + orientation. + + Relates to #37356 + +commit 1ed3407930eabc913fdf556828c2a0ab43dd78a1 +Author: Evgenia Badyanova +Date: Mon Feb 25 16:23:22 2019 -0500 + + Reduce garbage from allocations in deprecation logger (#38780) (#39370) + + 1. Setting length for formatWarning String to avoid AbstractStringBuilder.ensureCapacityInternal calls + 2. Adding extra check for parameter array length == 0 to avoid unnecessarily creating StringBuilder in LoggerMessageFormat.format + + Helps to narrow the performance gap in throughout for geonames benchmark (#37411) by 3%. For more details: https://github.com/elastic/elasticsearch/issues/37530#issuecomment-462758384 + + Relates to #37530 + Relates to #37411 + Relates to #35754 + +commit 5c7dd6f0ee46d85d48023710fd2b314fb2ab3ded +Author: Lee Hinman +Date: Mon Feb 25 13:26:41 2019 -0700 + + Set mappings when creating indices in SuggestSearchIT (#39323) + + * Set mappings when creating indices in SuggestSearchIT + + These tests don't test dynamic mapping, so they can use preset mappings. This + removes the possibility they may fail due to the mapping not being available + since mapping updates are asynchronous. + + Resolves #39315 + + * Wrap creates in assertAcked + +commit 926291aac8c01af6f709cc4fc48a4803eb522f5a +Author: Benjamin Trent +Date: Mon Feb 25 14:22:41 2019 -0600 + + [DATA-FRAME] Sort `GET` transforms and stats by ID (#39365) (#39369) + + * [Data-Frame] Sort `GET` transforms and stats by ID + + * removing unused import + +commit 0f29b89655361e6dfe2d5cf98fff103db0fca97c +Author: Nhat Nguyen +Date: Mon Feb 25 14:43:59 2019 -0500 + + Unmute FollowerFailOverIT#testFailOverOnFollower + + Relates #38633 + +commit e6a124c1184e4be2f80ba69221b7c4a0bef9dc0b +Author: James Baiera +Date: Mon Feb 25 14:39:34 2019 -0500 + + [Backport 7.x] Fix the OS sensing code in ClusterFormationTasks (#38457) + + This fixes a bug in the sensing of the current OS family in the test cluster + formation code. Previously all builds would assume every environment + was windows and would jump to using the windows zip build. This fixes + the OS sensing code as well as updates some tests to account for + different build flavors. + + Backport of #38457 + +commit 1897883adc489fdbad1061e846b903791a2ff44a +Author: Hendrik Muhs +Date: Mon Feb 25 19:08:26 2019 +0100 + + [ML-DataFrame] Dataframe access headers (#39289) (#39368) + + store user headers as part of the config and run transform as user + +commit bf058d6e4d2db220bbd1bfd5ffc470a28bf74e2f +Author: Mayya Sharipova +Date: Mon Feb 25 12:49:18 2019 -0500 + + Fix anaylze NullPointerException when AnalyzeTokenList tokens is null (#39332) (#39361) + +commit 3d4952372670a1d99193fc444c949b56ec6ec919 +Author: Benjamin Trent +Date: Mon Feb 25 11:19:49 2019 -0600 + + [DATA-FRAME] adds specs and yml tests for existing endpoints (#39326) (#39363) + + * [DATA-FRAME] adds specs and yml tests for existing endpoints + + * removing bad URL, adding test for _all + +commit 48219112e34f1107d98eb37d047e86f24593d394 +Author: Nhat Nguyen +Date: Mon Feb 25 10:57:54 2019 -0500 + + Do not wait for advancement of checkpoint in recovery (#39006) + + With this change, we won't wait for the local checkpoint to advance to + the max_seq_no before starting phase2 of peer-recovery. We also remove + the sequence number range check in peer-recovery. We can safely do these + thanks to Yannick's finding. + + The replication group to be used is currently sampled after indexing + into the primary (see `ReplicationOperation` class). This means that + when initiating tracking of a new replica, we have to consider the + following two cases: + + - There are operations for which the replication group has not been + sampled yet. As we initiated the new replica as tracking, we know that + those operations will be replicated to the new replica and follow the + typical replication group semantics (e.g. marked as stale when + unavailable). + + - There are operations for which the replication group has already been + sampled. These operations will not be sent to the new replica. However, + we know that those operations are already indexed into Lucene and the + translog on the primary, as the sampling is happening after that. This + means that by taking a snapshot of Lucene or the translog, we will be + getting those ops as well. What we cannot guarantee anymore is that all + ops up to `endingSeqNo` are available in the snapshot (i.e. also see + comment in `RecoverySourceHandler` saying `We need to wait for all + operations up to the current max to complete, otherwise we can not + guarantee that all operations in the required range will be available + for replaying from the translog of the source.`). This is not needed, + though, as we can no longer guarantee that max seq no == local + checkpoint. + + Relates #39000 + Closes #38949 + + Co-authored-by: Yannick Welsch + +commit 236db51d343c76887373f840c7430fc9635c7da7 +Author: David Turner +Date: Mon Feb 25 16:44:13 2019 +0000 + + Fix testSnapshotFileFailureDuringSnapshot (#39362) + + Today this test catches an exception and asserts that its proximate cause has + message `Random IOException` but occasionally this exception is wrapped two + layers deep, causing the test to fail. This commit adjusts the test to look at + the root cause of the exception instead. + + 1> [2019-02-25T12:31:50,837][INFO ][o.e.s.SharedClusterSnapshotRestoreIT] [testSnapshotFileFailureDuringSnapshot] --> caught a top level exception, asserting what's expected + 1> org.elasticsearch.snapshots.SnapshotException: [test-repo:test-snap/e-hn_pLGRmOo97ENEXdQMQ] Snapshot could not be read + 1> at org.elasticsearch.snapshots.SnapshotsService.snapshots(SnapshotsService.java:212) ~[main/:?] + 1> at org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:135) ~[main/:?] + 1> at org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.masterOperation(TransportGetSnapshotsAction.java:54) ~[main/:?] + 1> at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:127) ~[main/:?] + 1> at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.doRun(TransportMasterNodeAction.java:208) ~[main/:?] + 1> at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[main/:?] + 1> at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?] + 1> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_202] + 1> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_202] + 1> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202] + 1> Caused by: org.elasticsearch.snapshots.SnapshotException: [test-repo:test-snap/e-hn_pLGRmOo97ENEXdQMQ] failed to get snapshots + 1> at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getSnapshotInfo(BlobStoreRepository.java:564) ~[main/:?] + 1> at org.elasticsearch.snapshots.SnapshotsService.snapshots(SnapshotsService.java:206) ~[main/:?] + 1> ... 9 more + 1> Caused by: java.io.IOException: Random IOException + 1> at org.elasticsearch.snapshots.mockstore.MockRepository$MockBlobStore$MockBlobContainer.maybeIOExceptionOrBlock(MockRepository.java:275) ~[test/:?] + 1> at org.elasticsearch.snapshots.mockstore.MockRepository$MockBlobStore$MockBlobContainer.readBlob(MockRepository.java:317) ~[test/:?] + 1> at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.readBlob(ChecksumBlobStoreFormat.java:101) ~[main/:?] + 1> at org.elasticsearch.repositories.blobstore.BlobStoreFormat.read(BlobStoreFormat.java:90) ~[main/:?] + 1> at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getSnapshotInfo(BlobStoreRepository.java:560) ~[main/:?] + 1> at org.elasticsearch.snapshots.SnapshotsService.snapshots(SnapshotsService.java:206) ~[main/:?] + 1> ... 9 more + + FAILURE 0.59s J0 | SharedClusterSnapshotRestoreIT.testSnapshotFileFailureDuringSnapshot <<< FAILURES! + > Throwable #1: java.lang.AssertionError: + > Expected: a string containing "Random IOException" + > but: was "[test-repo:test-snap/e-hn_pLGRmOo97ENEXdQMQ] failed to get snapshots" + > at __randomizedtesting.SeedInfo.seed([B73CA847D4B4F52D:884E042D2D899330]:0) + > at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20) + > at org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testSnapshotFileFailureDuringSnapshot(SharedClusterSnapshotRestoreIT.java:821) + > at java.lang.Thread.run(Thread.java:748) + +commit 11fe8cd16f7b6f590e00b47d020fe3c445c24cab +Author: Marios Trivyzas +Date: Mon Feb 25 17:26:15 2019 +0100 + + [Tests] Fix flakiness by ensuring stable cluster (#39300) (#39356) + + In integration tests where `setBootstrapMasterNodeIndex()` is used in + combination with `autoMinMasterNodes = false` the cluster can start + bootstrapping once the number of nodes set with the + `setBootstrapMasterNodeIndex` have been started but it's not ensured + that all nodes have successfully joined to form the cluster. + + This behaviour was introduced with 5db7ed22a0aab6e630a89952ec94327d205fc3a3 + and in order to ensure that the cluster is properly formed before proceeding + with the integration test, use `ensureStableCluster()` with the + appropriate number of expected nodes. + + Fixes: #39220 + +commit b6734b412d27532e5a3c46a97a0e4c4add5af4d6 +Author: Luca Cavanna +Date: Mon Feb 25 16:41:02 2019 +0100 + + [DOCS] Fix typo in network-host.asciidoc + +commit 1d902cce88d1875cbfaa7ee0c668ef89493b439f +Author: Darren Meiss +Date: Mon Feb 25 10:38:38 2019 -0500 + + Edits to text in Optimistic Concurrency Ctrl doc (#39191) + +commit 8f0d864ae18036d3fd9296a02761a2608e3cce10 +Author: Darren Meiss +Date: Mon Feb 25 10:38:10 2019 -0500 + + Minor edits to text in Reindex API doc (#39137) + +commit dc23be5a9dd6843342650bff20baf33cbf787bcc +Author: David Turner +Date: Mon Feb 25 13:16:55 2019 +0000 + + Avoid creating a green index in RetentionLeaseIT (#39347) + + In #39224 we made shard history retention lease syncing ignore the + `index.write.wait_for_active_shards` setting on the index, and added a test + that showed that it was ignored. However the test as merged actually creates a + green index, so the `wait_for_active_shards` setting has no effect. This change + adjusts the test to create a yellow index to verify that + `wait_for_active_shards` really is ignored. + +commit 6f69ef165b923296d09cdcb9542e8dab1d3ee9d3 +Author: Martijn van Groningen +Date: Mon Feb 25 13:39:32 2019 +0100 + + Protect against the leader index being removed (#39351) + + when dealing with TimeoutException + + The `IndexFollowingIT#testDeleteLeaderIndex()`` test failed, + because a NPE was captured as fatal error instead of an IndexNotFoundException. + + Closes #39308 + +commit 9d97f3289d4b83e1ea18f1ed09bc9fcc21ad3b80 +Author: Costin Leau +Date: Mon Feb 25 14:04:00 2019 +0200 + + Mute CcrRollingUpgradeIT#testCannotFollowLeaderInUpgradedCluster + + See #39355 + +commit ea4e9ed0a921e10d393931644ab0d08e2d5b29a5 +Author: Martijn van Groningen +Date: Mon Feb 25 12:48:11 2019 +0100 + + Add missing tests for CcrRequestConverters (#39228) + +commit 9bf05388784bd9504b8dbec11db91f96255381f6 +Author: Martijn van Groningen +Date: Mon Feb 25 10:43:44 2019 +0100 + + Wait for index following is active for auto followed index (#39175) + + before executing pause follow api: + + https://github.com/elastic/elasticsearch/issues/39126#issuecomment-465512002 + + Closes #39126 + +commit a2bc41621cd8c6ba564fd6bfb17cd21b27074bea +Author: Yannick Welsch +Date: Mon Feb 25 18:24:06 2019 +0900 + + Clean GatewayAllocator when stepping down as master (#38885) + + This fixes an issue where a messy master election might prevent shard allocation to properly + proceed. I've encountered this in failing CI tests when we were bootstrapping multiple nodes. Tests + would sometimes time out on an `ensureGreen` after an unclean master election. The reason for + this is how the async shard information fetching works and how the clean-up logic in + GatewayAllocator is integrated with the rest of the system. When a node becomes master, it will, as + part of the first cluster state update where it becomes master, already try allocating shards (see + `JoinTaskExecutor`, in particular the call to `reroute`). This process, which runs on the + MasterService thread, will trigger async shard fetching. If the node is still processing an earlier + election failure in ClusterApplierService (e.g. due to a messy election), that will possibly trigger the + clean-up logic in GatewayAllocator after the shard fetching has been initiated by MasterService, + thereby cancelling the fetching, which means that no subsequent reroute (allocation) is triggered + after the shard fetching results return. This means that no shard allocation will happen unless the + user triggers an explicit reroute command. The bug imo is that GatewayAllocator is called from both + MasterService and ClusterApplierService threads, with no clear happens-before relation. The fix + here makes it so that the clean-up logic is also run on the MasterService thread instead of the + ClusterApplierService thread, reestablishing a clear happens-before relation. Note that testing this + is tricky. With the newly added test, I can quite often reproduce this by adding `Thread.sleep(10);` + in ClusterApplierService (to make sure it does not go too quickly) and adding `Thread.sleep(50);` in + `TransportNodesListGatewayStartedShards` to make sure that shard state fetching does not go too + quickly either. + + Note that older versions of Zen discovery are affected by this as well, but did not exhibit this issue + as often because master elections are much slower there. + +commit 7021e1bd3be866ed4af281bf0209883c02961f23 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Feb 25 20:35:08 2019 +1100 + + Add await busy loop for SimpleKdcLdapServer initialization (#39221) (#39342) + + There have been intermittent failures where either + LDAP server could not be started or KDC server could + not be started causing failures during test runs. + + `KdcNetwork` class from Apache kerby project does not set reuse + address to `true` on the socket so if the port that we found to be free + is in `TIME_WAIT` state it may fail to bind. As this is an internal + class for kerby, I could not find a way to extend. + + This commit adds a retry loop for initialization. It will keep + trying in an await busy loop and fail after 10 seconds if not + initialized. + + Closes #35982 + +commit 96c09b032dc637b7042ed5a8d91da91e3979094c +Author: David Turner +Date: Mon Feb 25 08:08:02 2019 +0000 + + Ignore waitForActiveShards when syncing leases (#39224) + + Adjust the retention lease sync actions so that they do not respect the + `index.write.wait_for_active_shards` setting on an index, allowing them to sync + retention leases even if insufficiently many shards are currently active to + accept writes. + + Relates #39089 + +commit 50d2736746f32e60d7b7ec213c6c39f9baa53b64 +Author: Armin Braun +Date: Mon Feb 25 09:15:19 2019 +0100 + + Fix Deadlock from Thread.suspend in Test (#39261) (#39341) + + * The lambda invoked by the `lockedExecutor` eventually gets JITed (which runs a static initializer that we will suspend in with a very tiny chance). + * Fixed by creating the `Runnable` in the main test thread and using the same instance in all threads + * Closes #35686 + +commit f17d408fbb98b0087b9d75db338fb73434f636c2 +Author: Nhat Nguyen +Date: Sat Feb 23 12:39:43 2019 -0500 + + Add cause to assert_no_failure when replay translog (#39333) + + We tripped this assertion three times for the last two weeks. However, + it only says "this IndexWriter is closed" without the actual cause. + + ``` + [2019-02-14T11:46:31,144][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [node-1] fatal error in thread [elasticsearch[node-1][generic][T#2]], exiting + + java.lang.AssertionError: unexpected failure while replicating translog entry: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed + ``` + + This change replaces an assert with an AssertionError so that we will + have the actual cause in the next build failures. + + Relates #38898 + +commit e80284231df7066414ded3fcbb6941899879fe7d +Author: Mayya Sharipova +Date: Sat Feb 23 11:52:43 2019 -0500 + + Backport distance functions vectors (#39330) + + Distance functions for dense and sparse vectors + + Backport for #37947, #39313 + +commit 6e06f821060c8974450dd4bc3d8a87b531e51cab +Author: Jason Tedor +Date: Fri Feb 22 18:34:33 2019 -0500 + + Fix failing CCR retention lease test + + Finally! This commit should fix the issues with the CCR retention lease + that has been plaguing build failures. The issue here is that we are + trying to prevent the clear session requests from being executed until + after we have been able to validate that retention leases are being + renewed. However, we were only blocking the clear session requests but + not blocking them when they are proxied through another node. This + commit addresses that. + + Relates #39268 + +commit 2d4c98a991a8e13c11296a8d285ca5fc63eab4ad +Author: Jason Tedor +Date: Fri Feb 22 18:16:28 2019 -0500 + + Change sort order of shard stats in CCR test + + This commit changes the sort order of shard stats that are collected in + CCR retention lease integration tests. This change is done so that + primaries appear first in sort order. + +commit e569cf8324e3d2b81b7e5cc7b736728697a942a0 +Author: Jason Tedor +Date: Fri Feb 22 18:03:17 2019 -0500 + + Address failing CCR retention lease test + + This test fails rarely but it is flaky in its current form. The problem + here is that we lack a guarantee on the retention leases having been + synced to all shard copies. We need to sleep long enough to ensure that + that occurs, and then we can sample the retention leases, possibly sleep + again (we usually will not have too since the first sleep will have been + long enough to allow a sync and a renewal to happen, if one was going to + happen), and the sample the retention leases for comparison. + + Closes #39331 + +commit e4e96b8181f35a9a674cbb425f310ad820b65902 +Author: Jason Tedor +Date: Fri Feb 22 17:31:45 2019 -0500 + + Fix shard logged in background lease renewal + + The shard logged here is the leader shard but it should be the follower + shard since this background retention lease renewal is happening on the + follower side. This commit fixes that. + +commit feb25c71a03588c96473be5f0960eb88b0381be3 +Author: Jason Tedor +Date: Fri Feb 22 17:22:30 2019 -0500 + + Simplify mocking in CCR retention lease tests + + This commit simplifies the use of transport mocking in the CCR retention + lease integration tests. Instead of adding a send rule between nodes, we + add a default send rule. This greatly simplifies the code here, and + speeds the test up a little bit too. + +commit c7516b03b679deffd7f2922c1c52a6c44dc9972a +Author: Zachary Tong +Date: Fri Feb 22 15:25:26 2019 -0500 + + Better HoltWinters parameter validation (#38747) + + We validate HW parameters (namely, window > 2 * period) when parsing + the XContent... but that means transport clients can configure bad + params. + + This change allows model to validate the window and throw an + exception if they wish. + + It also makes some test changes: + + - removes testBadModelParams(), which was a junk test (didn't do + anything), and bad param checking is done elsewhere in units tests + - Fixes one of the windows in testHoltWintersNotEnoughData() + - Ensures the period in testHoltWintersNotEnoughData() is >> window + - Removes `setTypes()` since that's deprecated + +commit 931953a3ee3dffb5fb6f1438fcb8bc628015b59f +Author: Tim Brooks +Date: Thu Feb 21 20:03:24 2019 -0700 + + Ensure index commit released when testing timeouts (#39273) + + This fixes #39245. Currently it is possible in this test that the clear + session call times-out. This means that the index commit will not be + released and there will be an assertion triggered in the test teardown. + This commit ensures that we wipe the leader index in the test to avoid + this assertion. + + It is okay if the clear session call times-out in normal usage. This + scenario is unavoidable due to potential network issues. We have a local + timeout on the leader to clean it up when this scenario happens. + +commit 3262d6c9171a4a7b691719735bc5954fc2602045 +Author: Benjamin Trent +Date: Fri Feb 22 10:55:38 2019 -0600 + + [ML-DataFrame] Add _preview endpoint (#38924) (#39319) + + * [DATA-FRAME] add preview endpoint + + * adjusting preview tests and fixing parser + + * adjusing preview transport + + * remove unused import + + * adjusting test + + * Addressing PR comments + + * Fixing failing test and adjusting for pr comments + + * fixing integration test + +commit 4f2bd238d255376c302e921da4f16a83b85958e1 +Author: David Roberts +Date: Fri Feb 22 15:22:38 2019 +0000 + + [ML] Increase datafeed integration test timeout for slow machines (#39311) + + The assertBusy() that waits the default 10 seconds for a + datafeed to complete very occasionally times out on slow + machines. This commit increases the timeout to 60 seconds. + It will almost never actually take this long, but it's + better to have a timeout that will prevent time being + wasted looking at spurious test failures. + +commit ca630bbe6f7d2973302aee7b6fa9f3468ead7570 +Author: Albert Zaharovits +Date: Fri Feb 22 15:57:09 2019 +0200 + + Fix DissectParserTests expecting unique keys (#39262) + + Fixes a bug in DissectParserTests where the tests expected dissect + keys to be unique but were not. + + Closes #39244 + +commit 2ad1e6aedc83820c759e6dcfb6b9ef8391a88103 +Author: Gordon Brown +Date: Fri Feb 22 08:03:36 2019 -0700 + + Fix testCannotShrinkLeaderIndex (#38529) + + This test should no longer pass when the functionality it is intended to + test is broken, as it now indexes a number of documents and verifies + that the index is staying on the same step until after indexing and + replication of those documents is finished. This prevents the test from + passing if the leader index progresses in its lifecycle during that time. + +commit 1c6818fe744122b1ea2dc300a4ceb9bbeacdf8a1 +Author: Dimitris Athanasiou +Date: Fri Feb 22 16:15:11 2019 +0200 + + [ML] Improve DeleteExpiredDataIT failure message (#39298) (#39310) + + This test failed once in a very long time with the assertion + that there is no document for the `non_existing_job` in the + state index. I could not see how that is possible and I cannot + reproduce. With this commit the failure message will reveal + some examples of the left behind docs which might shed a light + about what could go wrong. + +commit 9fea21aca5fc3b25504bf599487f07910c11301c +Author: Daniel Mitterdorfer +Date: Fri Feb 22 14:03:29 2019 +0100 + + Remove ExceptionsHelper#detailedMessage in tests (#37921) (#39297) + + With this commit we remove all usages of the deprecated method + `ExceptionsHelper#detailedMessage` in tests. We do not address + production code here but rather in dedicated follow-up PRs to keep the + individual changes manageable. + + Relates #19069 + +commit 401226fc908ae86fa6b2d291b823b09292e8fe22 +Author: Ioannis Kakavas +Date: Fri Feb 22 13:26:37 2019 +0200 + + Mute rolling upgrade watcher CRUD tests (#39293) + + This fails on old_cluster but mixed_cluster and upgraded_cluster + depend on watches set in old_cluster so that can't be muted on its + own + + Relates: https://github.com/elastic/elasticsearch/issues/33185 + +commit 42e0d63b36d4a0f012efa9ba7944bac83aed8ab0 +Author: Daniel Mitterdorfer +Date: Thu Feb 21 17:14:40 2019 +0100 + + Align generated release notes with doc standards (#39234) + + With this commit we change the script that generates the release notes + for Elasticsearch to align with the docs standards in the following + aspects: + + * The release notes title changes to include the product name + * The link text to the breaking changes page is explicitly defined + + Relates #39155 + +commit 3401afdf3592c8ff855935851d9c17b892e31e50 +Author: Lee Hinman +Date: Thu Feb 21 21:42:53 2019 -0700 + + Add ILM plugin for MonitoringIT tests (#39271) + + Without this, when creating the watch history indices they complain about there + being no such setting as `index.lifecycle.name`. + + Relates to #38805 + +commit 29243f70014ec6186ce179992307d8c3e198e102 +Author: Julie Tibshirani +Date: Thu Feb 21 17:57:46 2019 -0800 + + Avoid using TimeWarp in TransformIntegrationTests. (#39277) + + This commit makes `TransformIntegrationTests` into a standard integration test, as + opposed to using `TimeWarp`, which registers the mock component + `ScheduleEngineTriggerMock` to trigger watches. + + The simplification may help with flakiness we've observed `TimeWarp, as in #37882. + +commit 697911c31d324da36a0ada95a91e16f79b3ff484 +Author: Jay Modi +Date: Thu Feb 21 14:24:24 2019 -0700 + + Fixed missed stopping of SchedulerEngine (#39193) + + The SchedulerEngine is used in several places in our code and not all + of these usages properly stopped the SchedulerEngine, which could lead + to test failures due to leaked threads from the SchedulerEngine. This + change adds stopping to these usages in order to avoid the thread leaks + that cause CI failures and noise. + + Closes #38875 + +commit 44df76251f86388e18e2427969a5b9020f1bf3fa +Author: Tim Brooks +Date: Thu Feb 21 14:00:39 2019 -0700 + + Rebuild remote connections on profile changes (#39146) + + Currently remote compression and ping schedule settings are dynamic. + However, we do not listen for changes. This commit adds listeners for + changes to those two settings. Additionally, when those settings change + we now close existing connections and open new ones with the settings + applied. + + Fixes #37201. + +commit 34d06471c33b3360db0ab74a95ef291d6e5b778a +Author: Benjamin Trent +Date: Thu Feb 21 14:17:03 2019 -0600 + + [CI] Mute CcrRetentionLeaseIT.testRetentionLeaseIsRenewedDuringRecovery (#39270) + +commit 8072543428ae7440516fdd121543437ca30ed643 +Author: Benjamin Trent +Date: Thu Feb 21 13:43:09 2019 -0600 + + Muting AutoFollowIT.testAutoFollowManyIndices (#39265) + +commit 8af0e7c4b61e92ee5dfa0f7c7c49114be3db7ff6 +Author: Zachary Tong +Date: Thu Feb 21 14:18:45 2019 -0500 + + Only create final MatrixStatsResults on final reduction (#39205) + + MatrixStatsResults is the "final" result object, and runs an additional + computation in it's ctor to calculate covariance, etc. This means + it should only run on the final reduction instead of on every reduce. + +commit b9f8be6968596287bdc2cd4d372d4b650ae522f6 +Author: Jason Tedor +Date: Thu Feb 21 14:04:01 2019 -0500 + + Clarify the use of sleep in CCR test + + Sleeps in tests smell funny, and we try to avoid them to the extent + possible. We are using a small one in a CCR test. This commit clarifies + the purpose of that sleep by adding a comment explaining it. We also + removed a hard-coded value from the test, that if we ever modified the + value higher up where it was set, we could end up forgetting to change + the value here. Now we ensure that these would move in lock step if we + ever maintain them later. + +commit 719c38a36d9678c2b3561220a002f77dcb3d147e +Author: Jason Tedor +Date: Thu Feb 21 12:19:35 2019 -0500 + + Fix CCR tests that manipulate transport requests + + We have some CCR tests where we use mock transport send rules to control + the behavior that we desire in these tests. Namely, we want to simulate + an exception being thrown on the leader side, or a variety of other + situations. These send rules were put in place between the data nodes on + each side. However, it might not be the case that these requests are + being sent between data nodes. For example, a request that is handled on + a non-data master node would not be sent from a data node. And it might + not be the case that the request is sent to a data node, as it could be + proxied through a non-data coordinating node. This commit addresses this + by putting these send rules in places between all nodes on each side. + + Closes #39011 + Closes #39201 + +commit fc896e452c3c4e17e75d39c91a8f35dc36862668 +Author: Tanguy Leroux +Date: Thu Feb 21 18:08:06 2019 +0100 + + ReadOnlyEngine should update translog recovery state information (#39238) (#39251) + + `ReadOnlyEngine` never recovers operations from translog and never + updates translog information in the index shard's recovery state, even + though the recovery state goes through the `TRANSLOG` stage during + the recovery. It means that recovery information for frozen shards indicates + an unkown number of recovered translog ops in the Recovery APIs + (translog_ops: `-1` and translog_ops_percent: `-1.0%`) and this is confusing. + + This commit changes the `recoverFromTranslog()` method in `ReadOnlyEngine` + so that it always recover from an empty translog snapshot, allowing the recovery + state translog information to be correctly updated. + + Related to #33888 + +commit 0f68814e048351c34fb91283bc45c28958999b86 +Author: Mark Vieira +Date: Thu Feb 21 08:52:14 2019 -0800 + + Ensure global test seed is used for all random testing tasks (#38991) (#39195) + + This commit fixes a bug which resulted in every RandomizedTestingTask + generating its own unique test seed rather than using the global test + seed generated by the the RandomizedTestingPlugin. + + The issue didn't affect build invocations which explicitly ran with + -Dtest.seed. In those cases, the Ant junit task correctly uses the + global seed. + + Since the Ant random testing target looks for an Ant project property + for determining the existing seed we now explicitly set this inside + RandomizedTestingPlugin. It just so happens that, incidentally, Ant will + inherit system properties, thus wy the -Dtest.seed mechanism continued + to work. + +commit 4b77d0434aac3a28f76c3ce09d371353119f3846 +Author: Christoph Büscher +Date: Thu Feb 21 16:47:02 2019 +0100 + + Remove `nGram` and `edgeNGram` token filter names (#39070) + + In #30209 we deprecated the camel case `nGram` filter name in favour of `ngram` and + did the same for `edgeNGram` and `edge_ngram` and we are removing those names in + 8.0. This change disallows using the deprecated names for new indices created in 7.0 by + throwing an error if these filters are used. + + Relates to #38911 + +commit 08ad740d4805e36988477d1f8b93f95d5cfeb131 +Author: Albert Zaharovits +Date: Thu Feb 21 17:30:01 2019 +0200 + + Mute test (#39248) + + Mute test DissectParserTests.testBasicMatchUnicode + +commit 1a21cc035745a0a5cca81b13aaf5dd3065292da1 +Author: Armin Braun +Date: Thu Feb 21 16:27:18 2019 +0100 + + Simplify and Fix Synchronization in InternalTestCluster (#39168) (#39241) + + * Simplify and Fix Synchronization in InternalTestCluster (#39168) + + * Remove unnecessary `synchronized` statements + * Make `Predicate`s constants where possible + * Cleanup some stream usage + * Make unsafe public methods `synchronized` + * Closes #37965 + * Closes #37275 + * Closes #37345 + +commit d9de8993160137d4ec20453bfb45d39785d5f283 +Author: Lee Hinman +Date: Thu Feb 21 08:00:03 2019 -0700 + + Wrap accounting breaker check in assertBusy (#39211) + + There may be situations where indices have not yet been closed from a Lucene + perspective, causing the breaker to not immediately be at 0 + + Relates to #30290 + +commit ef921fd15779e832767e8842ebb5b7da59080459 +Author: Daniel Mitterdorfer +Date: Thu Feb 21 15:21:44 2019 +0100 + + Migrate Streamable to Writeable for cluster block package (#37391) (#39236) + +commit ecfd48b6d38f6a958a81b67029f0ac6ac258eb60 +Author: Marios Trivyzas +Date: Thu Feb 21 14:30:59 2019 +0200 + + [Tests] Make testEngineGCDeletesSetting deterministic (#38942) (#39231) + + `InternalEngine.resolveDocVersion()` uses `relativeTimeInMillis()` from + `ThreadPool` so it needs, the cached time to be advanced. Add a check + to ensure that and decrease the `thread_pool.estimated_time_interval` + to 1msec to prevent long running times for the test. + + Fixes: #38874 + + Co-authored-by: Boaz Leskes + +commit 1316825f52b42f0b73197e9532a51cf33a3ea34f +Author: Marios Trivyzas +Date: Thu Feb 21 12:42:54 2019 +0200 + + Replace superfluous usage of Counter with Supplier (#39048) (#39225) + + `Counter` was used as a means of a functional argument to pass + the relative cached time before `Supplier` iface was introduced. + +commit be8a5315d7033f52eada32eafb2e650824bfbd6d +Author: Ignacio Vera +Date: Thu Feb 21 11:09:50 2019 +0100 + + Extend nextDoc to delegate to the wrapped doc-value iterator for date_nanos (#39176) + + The type date_nanos does not direct doc-value iterators and it needs to extend `next_doc` in order to delegate the call to the wrapped iterator. + +commit f40139c403a46b724ab29e1534a70c514ed61c90 +Author: Martijn van Groningen +Date: Thu Feb 21 09:10:12 2019 +0100 + + Change ShardFollowTask to reuse common serialization logic (#39094) + + Initially in #38910, ShardFollowTask was reusing ImmutableFollowParameters' + serialization logic. After merging, bwc tests failed sometimes and + the binary serialization that ShardFollowTask was originally was using + was added back. ImmutableFollowParameters is using optional fields (optional vint) + while ShardFollowTask was not (vint). + +commit c28e1b2299105e92ae707895810e384fea2f9305 +Author: Martijn van Groningen +Date: Thu Feb 21 08:57:22 2019 +0100 + + Disable bwc tests for #39094 + +commit b854c92696fe3a53dbe4958ef49ba5c99928ec83 +Author: Tal Levy +Date: Wed Feb 20 17:45:24 2019 -0800 + + add missing `test2` index in alias example (#39212) (#39214) + + * missing 'test2' index example (#39055) + + If I got the idea of aliases properly, I think that the index "test2" should have + a reference in the example above of the following sentence: + + " ... we associate the alias `alias1` to both `test` and `test2` ... " + + * add PUT test2 + + * Update aliases.asciidoc + + swap which is write/read + + Co-authored-by: Guilherme Ferreira + +commit 8150ca40f22166885c821ba2e17a0c51413a5fd6 +Author: Tal Levy +Date: Wed Feb 20 17:35:11 2019 -0800 + + mute test3MasterNodes2Failed + +commit 6e7b64377598dd9d78c386773a4b38e315702452 +Author: Ryan Ernst +Date: Wed Feb 20 17:07:11 2019 -0800 + + Deprecate fallback to java on PATH (#37990) + + Finding java on the path is sometimes confusing for users and + unexpected, as well as leading to a different java being used than a + user expects. This commit adds warning messages when starting + elasticsearch (or any tools like the plugin cli) and using java found + on the PATH instead of via JAVA_HOME. + +commit a96df5d209863240f52d915c091eb8dfd18c025c +Author: Nhat Nguyen +Date: Wed Feb 20 12:35:02 2019 -0500 + + Reduce refresh when lookup term in FollowingEngine (#39184) + + Today we always refresh when looking up the primary term in + FollowingEngine. This is not necessary for we can simply + return none for operations before the global checkpoint. + +commit cdec11c4eb048e0c11ac19d51108fe1b36d3b6d4 +Author: Nhat Nguyen +Date: Wed Feb 20 08:10:29 2019 -0500 + + Relax history check in ShardFollowTaskReplicationTests (#39162) + + The follower won't always have the same history as the leader for its + soft-deletes retention can be different. However, if some operation + exists on the history of the follower, then the same operation must + exist on the leader. This change relaxes the history check in + ShardFollowTaskReplicationTests. + + Closes #39093 + +commit 820ba8169e3d99b414e4dbd339e546071224eab6 +Author: Nhat Nguyen +Date: Wed Feb 20 08:08:27 2019 -0500 + + Add retention leases replication tests (#38857) + + This commit introduces the retention leases to ESIndexLevelReplicationTestCase, + then adds some tests verifying that the retention leases replication works + correctly in spite of the presence of the primary failover or out of order + delivery of retention leases sync requests. + + Relates #37165 + +commit 0b676f07f60f6bb77b425427d373b5522be16520 +Author: Tal Levy +Date: Wed Feb 20 16:11:29 2019 -0800 + + mute failing test in 20_mix_typeless_typeful + + awaits fix in #39198 + +commit a6ae146ccc155faa6824473a43007d516ece29ec +Author: Mirko Jotic +Date: Wed Feb 20 16:34:19 2019 -0500 + + Converting Derivative Pipeline Agg integration test into AggregatorTestsCase. (#38679) + + Replicates the majority of existing Derivative pipeline integration tests into + an AggregatorTestCase, with the goal of removing the integration + tests in the near future. + +commit bfc9a00d9f79af1ed9b7671f3f96239485a75769 +Author: Lisa Cawley +Date: Wed Feb 20 12:03:41 2019 -0800 + + [DOCS] Unset machine learning upgrade mode (#39149) + +commit 24ac9da276d1808b5fd5041665d349a6336a8a22 +Author: Mark Vieira +Date: Wed Feb 20 11:57:20 2019 -0800 + + Mute CCR retention test that is consistently failing locally and in CI + +commit cb7e3708bcb03d6244ee8a669e086253c4377ef6 +Author: Tal Levy +Date: Wed Feb 20 11:12:01 2019 -0800 + + Rollup jobs should be cleaned up before indices are deleted (#38930) (#39144) + + Rollup jobs should be stopped + deleted before the indices are removed. + It's possible for an active rollup job to issue a bulk request, the test + ends and the cleanup code deletes all indices. The in-flight bulk + request will then stall + error because the index no-longer exists... + but this process might take longer than the StopRollup timeout. + + Which means the test fails, and often fails several other tests since + the job is still active (e.g. other tests cannot create the same-named + job, or fail to stop the job in their cleanup because it's still stalled). + + This tends to knock over several tests before the bulk finally times + out and the job shuts down. + + Instead, we need to simply stop jobs first. Inflight bulks will resolve + quickly, and we can carry on with deleting indices after the jobs are + confirmed inactive. + + stop-job.asciidoc tended to trigger this issue because it executed + an async stop API and then exited, which setup the above situation. In + can and did happen with other tests though. As an extra precaution, + the doc test was modified to substitute in wait_for_completion + to help head off these issues too. + +commit af451459a578a407ca7711c025741ee87a540026 +Author: Jay Modi +Date: Wed Feb 20 11:37:26 2019 -0700 + + Fix failures in SessionFactoryLoadBalancingTests (#39154) + + This change aims to fix failures in the session factory load balancing + tests that mock failure scenarios. For these tests, we randomly shut + down ldap servers and bind a client socket to the port they were + listening on. Unfortunately, we would occasionally encounter failures + in these tests where a socket was already in use and/or the port + we expected to connect to was wrong and in fact was to one of the ldap + instances that should have been shut down. + + The failures are caused by the behavior of certain operating systems + when it comes to binding ports and wildcard addresses. It is possible + for a separate application to be bound to a wildcard address and still + allow our code to bind to that port on a specific address. So when we + close the server socket and open the client socket, we are still able + to establish a connection since the other application is already + listening on that port on a wildcard address. Another variant is that + the os will allow a wildcard bind of a server socket when there is + already an application listening on that port for a specific address. + + In order to do our best to prevent failures in these scenarios, this + change does the following: + + 1. Binds a client socket to all addresses in an awaitBusy + 2. Adds assumption that we could bind all valid addresses + 3. In the case that we still establish a connection to an address that + we should not be able to, try to bind and expect a failure of not + being connected + + Closes #32190 + +commit 3d93011e32011a6300c73eeff24de37b17acd5bd +Author: Igor Motov +Date: Tue Feb 19 16:02:11 2019 -0500 + + Fix median calculation in MedianAbsoluteDeviationAggregatorTests (#38979) + + Fixes an error in median calculation in + MedianAbsoluteDeviationAggregatorTests for odd number of sample points, + which causes some rare test failures. + + Fixes #38937 + +commit 90b1b36f502e331dac8ed2978a77143ce2f6fbe1 +Author: Jason Tedor +Date: Wed Feb 20 13:18:51 2019 -0500 + + Add cleanup logic to CCR retention lease test + + This commit adds some logic to remove the mock transport rules at the + end of a CCR retention lease test. + +commit 224600f3706217acff797b0c47d6558f2af404fa +Author: Jason Tedor +Date: Wed Feb 20 13:03:27 2019 -0500 + + Bump jackson-databind version for AWS SDK (#39183) + + This commit bumps the jackson-databind version for discovery-ec2 and + repository-s3 to 2.8.11.3. + +commit cfd7c77b64f9b9820bce557e799c62a0dfbbb17b +Author: Jason Tedor +Date: Wed Feb 20 12:03:45 2019 -0500 + + Fix broken CCR retention lease unfollow test + + This commit fixes a broken CCR retention lease unfollow test. The + problem with the test is that the random subset of shards that we picked + to disrupt would not necessarily overlap with the actual shards in + use. We could take a non-empty subset of [0, 3] (e.g., { 2 }) when the + only shard IDs in use were [0, 1]. This commit fixes this by taking into + account the number of shards in use in the test. + + With this change, we also take measure to ensure that a successful + branch is tested more frequently than would otherwise be the case. On + that branch, we want to sometimes pretend that the retention lease is + already removed. The randomness here was also sometimes selecting a + subset of shards that did not overlap with the shards actually in use + during the test. While this does not break the test, it is confusing and + reduces the amount of coverage of that branch. + + Relates #39185 + +commit d74e25a778a1b84475e0c101101da5c3a0294c41 +Author: Lisa Cawley +Date: Tue Feb 19 11:53:35 2019 -0800 + + [DOCS] Edits the remote clusters documentation (#38996) + +commit f1c8aa816f6b6a35dab7d04e670f9145e1cd389b +Author: Tal Levy +Date: Wed Feb 20 08:47:04 2019 -0800 + + fix dangling tag in TasksClientDocumentationIT (#39157) + + fix dangling tag in TasksClientDocumentationIT + and fix tag mentions from list-tasks to cancel-tasks + +commit 751c05eff98672663a531b6c9b59ab4e2eafcd4c +Author: Jason Tedor +Date: Wed Feb 20 11:38:42 2019 -0500 + + Bump jackson-databind version for ingest-geoip (#39182) + + This commit bumps the jackson-databind version for ingest-geoip to + 2.8.11.3. + +commit fe9b89ef9777058ec0d23344b240aff29d16a2b5 +Author: James Baiera +Date: Wed Feb 20 11:09:43 2019 -0500 + + Add an exception throw if waiting on transport port file fails (#37574) + + In the ClusterConfiguration class of the build source, there is an Ant waitfor block + that runs to ensure that the seed node's transport ports file is created before + trying to read it. If the wait times out, the file read fails with not much helpful info. + This just adds a timeout property to the waitfor block and throws a descriptive + exception instead. + + Backports #37574 + +commit af8ef1bb985b3d3a984c9399502e39cfe071a857 +Author: Albert Zaharovits +Date: Wed Feb 20 15:32:35 2019 +0200 + + Do not create the missing index when invoking getRole (#39039) + + In most of the places we avoid creating the `.security` index (or updating the mapping) + for read/search operations. This is more of a nit for the case of the getRole call, + that fixes a possible mapping update during a get role, and removes a dead if branch + about creating the `.security` index. + +commit 0594a467f28d043420814419c245d4a9ae16cd12 +Author: Martijn van Groningen +Date: Wed Feb 20 15:37:40 2019 +0100 + + Add support for ccr follow info api to HLRC. (#39115) + + This API was introduces after #33824 was closed. + +commit 48984f647d15159ba20d7783694349645167b472 +Author: Jason Tedor +Date: Wed Feb 20 09:46:11 2019 -0500 + + Mute failing CCR retention lease unfollow test + + This commit mutes a CCR retention lease unfollow test that is failing + randomly, but frequently. + +commit c78306980457135328987ab796eea62ce05b8d0e +Author: Ioannis Kakavas +Date: Wed Feb 20 15:35:35 2019 +0200 + + Fix NPE on Stale Index in IndicesService(#39173) + + This is a backport of #38891 which closes #38845 + +commit 09ea3ccd169db4e70743b53714f0b2e1b4a4d64e +Author: Jason Tedor +Date: Wed Feb 20 07:05:47 2019 -0500 + + Remove retention leases when unfollowing (#39088) + + This commit attempts to remove the retention leases on the leader shards + when unfollowing an index. This is best effort, since the leader might + not be available. + +commit eae2c9dd5c53eefefc786bb7777f360e40be8a6c +Author: Darren Meiss +Date: Wed Feb 20 04:36:37 2019 -0500 + + Edits to text in Phrase Suggester doc (#38966) + +commit 27f7ff157bc83bc6c3470629645ea4571b0bc89f +Author: Darren Meiss +Date: Wed Feb 20 04:35:17 2019 -0500 + + Fix a typo in the cat shards API docs (#38536) + +commit c28b6fb9b6c0e377a593f059ea0d11e4537baeda +Author: Adrien Grand +Date: Wed Feb 20 11:33:50 2019 +0100 + + Reenable test in `indices.put_mapping/20_mix_typeless_typeful.yml`. (#39056) (#39057) + + This test had been disabled because of test failures, but it only affected the + 6.x branch. The fix for 6.x is at #39054. On master/7.x/7.0 we can reenable the + test as-is. + +commit 988ce844e1be9261470ed1b9a6c86aaf20fcf612 +Author: Darren Meiss +Date: Wed Feb 20 04:41:17 2019 -0500 + + Edits to text in Bulk API doc (#39083) + +commit a070cd68221b5bfc462d575f2fac565b4adbc0ee +Author: Darren Meiss +Date: Wed Feb 20 04:40:47 2019 -0500 + + Edits to text in Multi Get API doc (#39082) + +commit ad792669bef37f66fd5f04f1078c7e7be1c9c630 +Author: Darren Meiss +Date: Wed Feb 20 04:40:16 2019 -0500 + + Edits to text in Update By Query API doc (#39078) + +commit 07ebb8c49a68f85bcac604382f447ee00d9b17f4 +Author: Darren Meiss +Date: Wed Feb 20 04:39:35 2019 -0500 + + Edits to text in Update API doc (#39069) + +commit 2f520c663cfecec6624b5b026521d7f904bec535 +Author: Darren Meiss +Date: Wed Feb 20 04:39:00 2019 -0500 + + Edits to text in Delete By Query API doc (#39017) + +commit 1037aa06658e82a49b013bba95f90ec6680a637b +Author: Darren Meiss +Date: Wed Feb 20 04:38:26 2019 -0500 + + Edits to text in Index API doc (#39010) + +commit dc1ed80c7374a764e930e3a585b1948b1a7ffc1a +Author: Darren Meiss +Date: Wed Feb 20 04:37:15 2019 -0500 + + Edits to text in API Conventions docs (#39001) + +commit c1018db4048c704655820005b7f175f0e0039e44 +Author: Andrei Stefan +Date: Wed Feb 20 11:25:25 2019 +0200 + + SQL: enforce JDBC driver - ES server version parity (#38972) + + (cherry picked from commit 822a21f29491f295b22dacd04b747781a69ffa61) + +commit 92206c85676e2158d781f8785280d49a25c35e38 +Author: Andrei Stefan +Date: Wed Feb 20 11:23:47 2019 +0200 + + Added "validate.properties" property to JDBC's list of allowed properties. (#39050) + + This defaults to "true" (current behavior) and will throw an exception + if there is a property that cannot be recognized. If "false", it will + ignore anything unrecognizable. + + (cherry picked from commit 38fbf9792bcf4fe66bb3f17589e5fe6d29748d07) + +commit efffb3d5b7ed4ffac4c90084a04b36e42065f60a +Author: David Turner +Date: Wed Feb 20 08:55:04 2019 +0000 + + Simplify calculation in AwarenessAllocationDecider (#38091) + + Today's calculation of the maximum number of shards per attribute is rather + convoluted. This commit clarifies that it returns + ceil(shardCount/numberOfAttributes). + +commit 00a26b9dd21a5c88eb773c2dc139353563b8630d +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Wed Feb 20 08:27:07 2019 +0100 + + Blob store compression fix (#39073) + + Blob store compression was not enabled for some of the files in + snapshots due to constructor accessing sub-class fields. Fixed to + instead accept compress field as constructor param. Also fixed chunk + size validation to work. + + Deprecated repositories.fs.compress setting as well to be able to unify + in a future commit. + +commit 4aa50ed34842ab2522c66e4fbf9d7daa9cfeb59c +Author: Tim Vernum +Date: Wed Feb 20 19:18:00 2019 +1100 + + Resolve concurrency with watcher trigger service (#39164) + + The watcher trigger service could attempt to modify the perWatchStats + map simultaneously from multiple threads. This would cause the + internal state to become inconsistent, in particular the count() + method may return an incorrect value for the number of watches. + + This changes replaces the implementation of the map with a + ConcurrentHashMap so that its internal state remains consistent even + when accessed from mutiple threads. + + Backport of: #39092 + +commit ec2b64af6342f8f93cf2e22df74a68e4de8090bb +Author: Ioannis Kakavas +Date: Tue Feb 19 19:31:25 2019 +0200 + + Disable date parsing test in non english locale (#39052) + + This ensures we do not attempt to parse non english locale dates + in FIPS mode. The error, originally assumed to affect only Joda, + affects Java time in the same manner and manifests only with the + version of BouncyCastle FIPS certified provider we use in tests. + The upstream issue https://github.com/bcgit/bc-java/issues/405 + indicates that the behavior is resolved in later versions of the + BouncyCastle library and should be tested again when the new + versions become FIPS 140 certified + +commit f5b28ca69d04aa46f2f83f102580806c9e356a54 +Author: Julie Tibshirani +Date: Tue Feb 19 17:30:25 2019 -0800 + + Enable test logging for TransformIntegrationTests#testSearchTransform. + + There is already fairly detailed debug logging in the watcher framework, which + should hopefully help debug the failure. + + Relates to #37882. + +commit 41ac6f9c55d8fe72fc38e71fdb88119ae861760e +Author: Lee Hinman +Date: Tue Feb 19 16:14:31 2019 -0700 + + Revert "Mute failing test 20_mix_typless_typefull (#38781)" (#38912) (#39141) + + Backport of #38912 + + This reverts commit b91e0589fe1efdaa5061a75a3674a5cc8706b703. + + This should be fixed by #38873 + + Resolves #38711 + +commit b5dbd1a027768fbfeeb899aef91ee1cbe8613254 +Author: Tal Levy +Date: Tue Feb 19 13:17:51 2019 -0800 + + AwaitsFix XPackUsageIT#testXPackCcrUsage. + + relates to #39126. + +commit 50b3858f7c6046cf4f59cf658d514935690cc53d +Author: Hendrik Muhs +Date: Tue Feb 19 17:36:03 2019 +0100 + + add version 6.6.2 + +commit f30f1fe9b67e0900f1e314caa49c8a67d1fa2758 +Author: Tal Levy +Date: Tue Feb 19 11:10:39 2019 -0800 + + fix RethrottleTests retry (#38978) (#39131) + + the RethrottleTests assumed that tasks that were + unprepared to rethrottle would bubble up into the + Rethrottle response as an ElasticsearchException + wrapping an IllegalArgumentException. This seems to + have changed to potentially involve further levels of + wrapping. + + This change makes the retry logic more resilient to + arbitrary nesting of the underlying IllegalArgumentException + +commit 109b6451fd0e276f652b04d9bbafec878b11903e +Author: Benjamin Trent +Date: Tue Feb 19 12:45:56 2019 -0600 + + ML refactor DatafeedsConfig(Update) so defaults are not populated in queries or aggs (#38822) (#39119) + + * ML refactor DatafeedsConfig(Update) so defaults are not populated in queries or aggs + + * Addressing pr feedback + +commit 80540a2fccc8d18ad3995e1643eeed2f0a488802 +Author: Lee Hinman +Date: Tue Feb 19 10:54:15 2019 -0700 + + Document 'max_size' parameter as shard size for rollover (#38750) + + It was not clear that this is *primary* shard size, not the entire shard size. + + Resolves #37981 + +commit 0a9574c9d49dfc8fdc9708cf3a6e0903a625c369 +Author: David Turner +Date: Tue Feb 19 17:52:25 2019 +0000 + + Add some missing toString() implementations (#39124) + + Sometimes we turn objects into strings for logging or debugging using + `toString()`, but the default implementation is often unhelpful. This change + improves on this in two places I ran into recently. + +commit 210f34f8e9149870cf9639d4efef92aeb331afac +Author: Ioannis Kakavas +Date: Tue Feb 19 17:43:24 2019 +0200 + + Remove BCryptTests (#39098) + + This test was added to verify that we fixed a specific behavior in + Bcrypt and hasn't been running for almost 4 years now. + +commit fef9bdb23ff1af0654f29577c8ba492c9ef6132c +Author: Jason Tedor +Date: Tue Feb 19 09:53:36 2019 -0500 + + Allow retention lease operations under blocks (#39089) + + This commit allows manipulating retention leases under blocks. + +commit 35e30b34f96bf200b54be1c42c364f71d76d2a1e +Author: David Roberts +Date: Tue Feb 19 15:11:10 2019 +0000 + + [ML] Stop the ML memory tracker before closing node (#39111) + + The ML memory tracker does searches against ML results + and config indices. These searches can be asynchronous, + and if they are running while the node is closing then + they can cause problems for other components. + + This change adds a stop() method to the MlMemoryTracker + that waits for in-flight searches to complete. Once + stop() has returned the MlMemoryTracker will not kick + off any new searches. + + The MlLifeCycleService now calls MlMemoryTracker.stop() + before stopping stopping the node. + + Fixes #37117 + +commit 12f696345606b3ae9a8070c9e3e1ec9786c5ad2b +Author: Jason Tedor +Date: Tue Feb 19 08:53:16 2019 -0500 + + Fix retention leases sync on recovery test + + This test had a bug. We attempt to allow only the primary to be + allocated, to force all replicas to recovery from the primary after we + had set the state of the retention leases on the primary. However, in + building the index settings, we were overwriting the settings that + exclude the replicas from being allocated. This means that some of the + replicas would end up assigned and rather than receive retention leases + during recovery, they would be part of the replication group receiving + retention leases as they are manipulated. Since retention lease renewals + are only synced periodically, this means that the replica could be + lagging a little behind in some cases leading to an assertion tripping + in the test. This commit addresses this by ensuring that the replicas + are indeed not allocated until after the retention leases are done being + manipulated on the replica. We did this by not overwriting the exclude + settings. + + Closes #39105 + +commit bbcdea43c54f7aa40c1298d7b34a6f185805fcff +Author: David Roberts +Date: Tue Feb 19 13:47:47 2019 +0000 + + [ML] Allow stop unassigned datafeed and relax unset upgrade mode wait (#39034) + + These two changes are interlinked. + + Before this change unsetting ML upgrade mode would wait for all + datafeeds to be assigned and not waiting for their corresponding + jobs to initialise. However, this could be inappropriate, if + there was a reason other that upgrade mode why one job was unable + to be assigned or slow to start up. Unsetting of upgrade mode + would hang in this case. + + This change relaxes the condition for considering upgrade mode to + be unset to simply that an assignment attempt has been made for + each ML persistent task that did not fail because upgrade mode + was enabled. Thus after unsetting upgrade mode there is no + guarantee that every ML persistent task is assigned, just that + each is not unassigned due to upgrade mode. + + In order to make setting upgrade mode work immediately after + unsetting upgrade mode it was then also necessary to make it + possible to stop a datafeed that was not assigned. There was + no particularly good reason why this was not allowed in the past. + It is trivial to stop an unassigned datafeed because it just + involves removing the persistent task. + +commit d8852b83d059227608e04d64787ea7183dd4208e +Author: Adrien Grand +Date: Tue Feb 19 15:03:47 2019 +0100 + + Don't swallow IOExceptions in InternalTestCluster. (#39068) + + Relates #39030 + +commit 7f8a6403636f9421f088966a7e050523d490d7ee +Author: Alexander Reelsen +Date: Tue Feb 19 14:11:52 2019 +0100 + + Fix DateFormatters.parseMillis when no timezone is given (#39100) + + The parseMillis method was able to work on formats without timezones by + falling back to UTC. The Date Formatter interface did not support this, as + the calling code was using the `Instant.from` java time API. + + This switches over to an internal method which adds UTC as a timezone. + + Closes #39067 + +commit 3cd6feeb0bea75444e0fa7e8773e4459fa416830 +Author: Dimitrios Liappis +Date: Tue Feb 19 13:58:47 2019 +0200 + + Remove unnecessary Dockerfile commands (#39095) + + As the Dockerfile evolved we don't need anymore certain commands like + `unzip`, `which` and `wget` allowing us to slightly shrink the images. + + Backport of: #39040 + +commit 4d820d56895604fbeb3fde3b5c7ecf14379b15cb +Author: David Turner +Date: Tue Feb 19 10:00:46 2019 +0000 + + repository-s3 also works with S3-compatibles (#38524) + + - Notes that you can adjust the `s3.client.*.endpoint` setting to point to a + repository held on an S3-compatible service. + - Notes that the default is `s3.amazonaws.com` and not to auto-detect the + endpoint. + - Reformats docs to width. + + Closes #35925 + +commit 2e2567e827de8012d141f1af5db00ee02748e007 +Author: Hendrik Muhs +Date: Tue Feb 19 10:53:50 2019 +0100 + + add debug info for intermittent test failure + +commit 83402b13203e1e5578d47fbe492d14ff4ca704b3 +Author: Jim Ferenczi +Date: Tue Feb 19 10:47:59 2019 +0100 + + Remove beta marker from the synonym_graph docs (#38185) + +commit 1ecdef707de32f54d8c7dc9b8ffd5cdba816c5ab +Author: Alpar Torok +Date: Tue Feb 19 11:15:35 2019 +0200 + + Make specifying java home mandatory for testclusters (#37864) + + Since we build and test with multiple versions it's better to be + explicit about it. + +commit c8d59f6f0f3ff54873e6cf8a756c02aa8b6bc2ea +Author: Martijn van Groningen +Date: Tue Feb 19 08:53:20 2019 +0100 + + Fix shard follow task startup error handling (#39053) + + Prior to this commit, if during fetch leader / follower GCP + a fatal error occurred, then the shard follow task was removed. + + This is unexpected, because if such an error occurs during the lifetime of shard follow task then replication is stopped and the fatal error flag is set. This allows the ccr stats api to report the fatal exception that has occurred (instead of the user grepping through the elasticsearch logs). + + This issue was found by a rare failure of the `FollowStatsIT#testFollowStatsApiIncludeShardFollowStatsWithRemovedFollowerIndex` test. + + Closes #38779 + +commit 199155f5fb94c7e006da9f58fe3a4c9268f71bfc +Author: Jim Ferenczi +Date: Tue Feb 19 08:52:24 2019 +0100 + + Enforce Completion Context Limit (#38675) (#39075) + + This change adds a limit to the number of completion contexts that a completion field can define. + + Closes #32741 + +commit 59e9a0f4f4ed9fb85ad55b74c5e76292f701de79 +Author: Ioannis Kakavas +Date: Tue Feb 19 08:45:01 2019 +0200 + + Disable specific locales for tests in fips mode (#38938) + + * Disable specific locales for tests in fips mode + + The Bouncy Castle FIPS provider that we use for running our tests + in fips mode has an issue with locale sensitive handling of Dates as + described in https://github.com/bcgit/bc-java/issues/405 + + This causes certificate validation to fail if any given test that + includes some form of certificate validation happens to run in one + of the locales. This manifested earlier in #33081 which was + handled insufficiently in #33299 + + This change ensures that the problematic 3 locales + + * th-TH + * ja-JP-u-ca-japanese-x-lvariant-JP + * th-TH-u-nu-thai-x-lvariant-TH + + will not be used when running our tests in a FIPS 140 JVM. It also + reverts #33299 + +commit 5c30446bd0ac60ffcc3a51408aebb74be024dab7 +Author: Albert Zaharovits +Date: Tue Feb 19 02:17:05 2019 +0200 + + Fix libs:ssl-config project setup (#39074) + + The build script file for the `:libs:elasticsearch-ssl-config` and + `:libs:ssl-config-tests` projects was incorrectly named `eclipse.build.gradle` + while the expected name was `eclipse-build.gradle`. + In addition, this also adds a missing snippet in the `build.gradle` conf file, + that fixes the project setup for Eclipse users. + +commit 6bc88b00eca791fcf4f73707f61fb5d3593c95aa +Author: Albert Zaharovits +Date: Tue Feb 19 00:24:35 2019 +0200 + + Mute GatewayMetaStateTests.testAtomicityWithFailures (#39079) + + Mute test GatewayMetaStateTests.testAtomicityWithFailures + +commit 2d8f6b6501466ebc2e0a064456986010962a6797 +Author: Jason Tedor +Date: Mon Feb 18 16:52:51 2019 -0500 + + Introduce retention lease state file (#39004) + + This commit moves retention leases from being persisted in the Lucene + commit point to being persisted in a dedicated state file. + +commit d43ac8fe118ee481173b657a21892159487ce0c3 +Author: Jason Tedor +Date: Mon Feb 18 15:06:54 2019 -0500 + + Include in log retention leases that failed to sync + + When retention leases fail to sync after an expiration check, we emit a + log message about this. This commit adds the retention leases that + failed to sync. + +commit bbb61002bae8e7901ddac092935b8fcfaea44ccd +Author: Jason Tedor +Date: Mon Feb 18 15:01:44 2019 -0500 + + Add some logging related to retention lease syncing (#39066) + + When the background retention lease sync fires, we check an see if any + retention leases are expired. If any did expire, we execute a full + retention lease sync (write action). Since this is happening on a + background thread, we do not block that thread waiting for success (it + will simply try again when the timer elapses). However, we were + swallowing exceptions that indicate failure. This commit addresses that + by logging the failures. Additionally, we add some trace logging to the + execution of syncing retention leases. + +commit 99b2bc3461eac54dc7ecdc0706bd43fabcc5f904 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Mon Feb 18 18:10:00 2019 +0100 + + Fix potential race during TcpTransport close (#39031) + + Fixed two potential causes for leaked threads during tests: + 1. When adding a channel to serverChannels, we add it under a monitor + that we do not use when reading from it. This is potentially unsafe if + there is no other happens-before relationship ensuring the safety of + this. + 2. Long-shot but if the thread pool was shutdown before entering this + code, we would silently forget about closing server channels so added + assert. + + Strengthened the locking to ensure that once we stop the transport, no + new server channels can be made. + + Relates to CI failure issue: #37543 + +commit ae9243ad0a80e5276cd3c74029b41bbc4a9720f2 +Author: David Roberts +Date: Mon Feb 18 17:37:30 2019 +0000 + + Reduce single node test cleanup logging (#39060) + + As per https://github.com/elastic/elasticsearch/pull/39049#discussion_r257719530 + +commit 231a8921610a5e4102d05259ddef56c5b07fc393 +Author: Darren Meiss +Date: Mon Feb 18 03:33:07 2019 -0500 + + Fix wording in Get API docs (#39012) + +commit 007aaf716a5067c103197cbc5258d78bb088e426 +Author: Darren Meiss +Date: Mon Feb 18 03:29:25 2019 -0500 + + Fix punctuation in replication docs (#39009) + +commit 231c8c6d4005fa1b0aa56ba585a4acb8578021e5 +Author: Darren Meiss +Date: Mon Feb 18 03:34:39 2019 -0500 + + Fix wording in Delete API docs (#39013) + +commit ce412908ed0ea2306fb23e3e53319e21715dfe81 +Author: Martijn van Groningen +Date: Mon Feb 18 16:14:40 2019 +0100 + + also check ccr stats api return empty response in ensureNoCcrTasks() + + If this fails then it returns more detailed information, for example + fatal error. + +commit ab4d5f404f7a773a053f6910baeddba6850c743b +Author: Alan Woodward +Date: Mon Feb 18 14:44:07 2019 +0000 + + Add overlapping, before, after filters to intervals query (#38999) + + Lucene recently added `overlapping`, `before` and `after` filters to the intervals package. This + commit exposes them in elasticsearch. + +commit 2947ccf5c34cc2b9b7a5f4d2d4f70d6c68e38627 +Author: Nhat Nguyen +Date: Mon Feb 18 09:10:44 2019 -0500 + + Add remote recovery to ShardFollowTaskReplicationTests (#39007) + + We simulate remote recovery in ShardFollowTaskReplicationTests + by bootstrapping the follower with the safe commit of the leader. + + Relates #35975 + +commit 1efb01661c430f2102ea2a92e459c2126c74d049 +Author: Hendrik Muhs +Date: Mon Feb 18 15:41:25 2019 +0100 + + set minimum supported version (#39043) (#39051) + + change the minimum supported version of data frame transform + +commit 4fd1f8048d1808f26ea391b1767ff429776e5484 +Author: Martijn van Groningen +Date: Mon Feb 18 15:23:33 2019 +0100 + + Mute test #38949 + +commit b660d2cac666542634f4eb6c921bc2064f270477 +Author: David Roberts +Date: Mon Feb 18 14:04:59 2019 +0000 + + [ML] More advanced post-test cleanup of ML indices (#39049) + + The .ml-annotations index is created asynchronously when + some other ML index exists. This can interfere with the + post-test index deletion, as the .ml-annotations index + can be created after all other indices have been deleted. + + This change adds an ML specific post-test cleanup step + that runs before the main cleanup and: + + 1. Checks if any ML indices exist + 2. If so, waits for the .ml-annotations index to exist + 3. Deletes the other ML indices found in step 1. + 4. Calls the super class cleanup + + This means that by the time the main post-test index + cleanup code runs: + + 1. The only ML index it has to delete will be the + .ml-annotations index + 2. No other ML indices will exist that could trigger + recreation of the .ml-annotations index + + Fixes #38952 + +commit e8ea85d6e9ec32c771a23b1d158814c9e5d22158 +Author: Martijn van Groningen +Date: Mon Feb 18 14:50:12 2019 +0100 + + wait for shard to be allocated before executing a resume follow api + +commit 45b17e864504a3df300f303bcc7b47c36f1f9f99 +Author: Adrien Grand +Date: Mon Feb 18 13:59:58 2019 +0100 + + Don't close caches while there might still be in-flight requests. (#38958) + + Many of our index components use ref-counting so that in the event that a shard + is closed while there are still ongoing requests, then the index reader and the + store only effectively get closed when ongoing requests have finished. However + we don't apply the same principle to the request and query caches, which might + get closed while there are still in-flight requests. + + This commit adds ref-counting to `IndicesService` so that the caches and other + components it maintains only get closed when all shards are effectively closed. + + Closes #37117 + +commit af7b89b80cbf6c224360d80812d44a3d57be06ab +Author: Alpar Torok +Date: Mon Feb 18 14:30:01 2019 +0200 + + Force kill testcluster nodes (#37353) + + * Force kill testcluster nodes + +commit d59ec89726dd40408d42f0b020a4dac08d115c54 +Author: Clinton Gormley +Date: Mon Feb 18 13:32:34 2019 +0100 + + Update track-total-hits.asciidoc + + Added missing ` + +commit 818d8951b85e76c44d8b9a0b391c2f1144a2abb8 +Author: Alpar Torok +Date: Mon Feb 18 14:10:10 2019 +0200 + + Make pullFixture a task dependency of resolveAllDependencies (#38956) + + * Make pullFixture a task dependency of resolveAllDependencies + + With this change we will pull the docker test fixtures ( and thus cache + the images ) for older versions too as the `resolveAllDependencies` is + already being called on the bwc checkouts too. + +commit 9b4d96534b0b2ee3fb590418c7e0ff86adb689f3 +Author: Martijn Laarman +Date: Mon Feb 18 12:40:07 2019 +0100 + + Fix #38623 remove xpack namespace REST API (#38625) (#39036) + + * Fix #38623 remove xpack namespace REST API + + Except for xpack.usage and xpack.info API's, this moves the last remaining API's out of the xpack namespace + + * rename xpack api's inside inside the files as well + + * updated yaml tests references to xpack namespaces api's + + * update callsApi calls in the IT subclasses + + * make sure docs testing does not use xpack namespaced api's + + * fix leftover xpack namespaced method names in docs/build.gradle + + * found another leftover reference + + (cherry picked from commit ccb5d934363c37506b76119ac050a254fa80b5e7) + +commit 9aa542fb1b10875f9c71333f55733e7fbd5a5309 +Author: Martijn van Groningen +Date: Mon Feb 18 12:02:34 2019 +0100 + + Mute test + + Relates to #38779 + +commit 4f662bd289e4bc5b97da14773a20875cadbae70c +Author: Hendrik Muhs +Date: Mon Feb 18 11:07:29 2019 +0100 + + Add data frame feature (#38934) (#39029) + + The data frame plugin allows users to create feature indexes by pivoting a source index. In a + nutshell this can be understood as reindex supporting aggregations or similar to the so called entity + centric indexing. + + Full history is provided in: feature/data-frame-transforms + +commit 8895befe511788ea1d7a3c3d51e0389e48d25cc3 +Author: Tim Vernum +Date: Mon Feb 18 20:07:22 2019 +1100 + + Generate mvn pom for ssl-config library (#39026) + + This is used by the reindex-client library which is published to maven + + Relates: #37287, #37527 + Backport of: #39019 + +commit ed08bc35374014bc4b56f92429b2374d51baee5b +Author: Martijn van Groningen +Date: Thu Feb 14 10:19:26 2019 +0100 + + Fix LocalIndexFollowingIT#testRemoveRemoteConnection() test (#38709) + + * During fetching remote mapping if remote client is missing then + `NoSuchRemoteClusterException` was not handled. + * When adding remote connection, check that it is really connected + before continue-ing to run the tests. + + Relates to #38695 + +commit 03b2ec6ee680f86d3a91fa78fb2d52aa760191ce +Author: Martijn van Groningen +Date: Mon Feb 18 09:06:30 2019 +0100 + + Test bi-directional index following during a rolling upgrade. (#38962) + + Follow index in follow cluster that follows an index in the leader cluster and another + follow index in the leader index that follows that index in the follow cluster. + + During the upgrade index following is paused and after the upgrade + index following is resumed and then verified index following works as expected. + + Relates to #38037 + +commit 408a800f743fd0b6c27f3d9b568e38139d8f01b9 +Author: Paul Sanwald +Date: Sun Feb 17 20:37:26 2019 -0500 + + ClusterClientIT refactor (#38872) (#39002) + + Add fixes for ClusterClientIT test and unmute tests. + +commit 204480d818008ee60388b6e6f54cf640861b7ee1 +Author: Nhat Nguyen +Date: Sun Feb 17 15:33:26 2019 -0500 + + Mute testRetentionLeaseIsRenewedDuringRecovery + + Tracked at #39011 + +commit a5ce1e0becf986c16e74d98d19f430bf26356e81 +Author: Jason Tedor +Date: Sat Feb 16 15:28:04 2019 -0500 + + Integrate retention leases to recovery from remote (#38829) + + This commit is the first step in integrating shard history retention + leases with CCR. In this commit we integrate shard history retention + leases with recovery from remote. Before we start transferring files, we + take out a retention lease on the primary. Then during the file copy + phase, we repeatedly renew the retention lease. Finally, when recovery + from remote is complete, we disable the background renewing of the + retention lease. + +commit b1c1daa63f2f1cec8ab41fb648489104544779df +Author: Tim Brooks +Date: Wed Feb 13 10:11:26 2019 -0700 + + Add get file chunk timeouts with listener timeouts (#38758) + + This commit adds a `ListenerTimeouts` class that will wrap a + `ActionListener` in a listener with a timeout scheduled on the generic + thread pool. If the timeout expires before the listener is completed, + `onFailure` will be called with an `ElasticsearchTimeoutException`. + + Timeouts for the get ccr file chunk action are implemented using this + functionality. Additionally, this commit attempts to fix #38027 by also + blocking proxied get ccr file chunk actions. This test being un-muted is + useful to verify the timeout functionality. + +commit d80325f288a4bf36b4fc2104add98d2eb714e125 +Author: Jason Tedor +Date: Sat Feb 16 12:27:06 2019 -0500 + + Mark fail over on follower test as awaits fix + + This test is failing since the introduction of recovery from + remote. This commit marks this test as awaits fix. + +commit a1a49f201de1ffeff509023c103115442a5cb3cc +Author: Luca Cavanna +Date: Fri Feb 15 13:44:55 2019 +0100 + + Tie break search shard iterator comparisons on cluster alias (#38853) + + `SearchShardIterator` inherits its `compareTo` implementation from `PlainShardIterator`. That is good in most of the cases, as such comparisons are based on the shard id which is unique, even when searching against indices with same names across multiple clusters (thanks to the index uuid being different). In case though the same cluster is registered multiple times with different aliases, the shard id is exactly the same, hence remote results will be returned before local ones with same shard id objects. That is because remote iterators are added before local ones, and we use a stable sorting method in `GroupShardIterators` constructor. + + This PR enhances `compareTo` for `SearchShardIterator` to tie break on cluster alias and introduces consistent `equals` and `hashcode` methods. This allows to remove a TODO in `SearchResponseMerger` which otherwise has to handle this special case specifically. Also, while at it I added missing tests around equals/hashcode and compareTo and expanded existing ones. + +commit 7e20a928886c8536f88bad654ed773214f296aac +Author: Nhat Nguyen +Date: Fri Feb 15 13:42:02 2019 -0500 + + Advance max_seq_no before add operation to Lucene (#38879) + + Today when processing an operation on a replica engine (or the + following engine), we first add it to Lucene, then add it to translog, + then finally marks its seq_no as completed. If a flush occurs after step1, + but before step-3, the max_seq_no in the commit's user_data will be + smaller than the seq_no of some documents in the Lucene commit. + +commit 20755e666ccb28808cdb60b470a33211c9743c68 +Author: Nhat Nguyen +Date: Fri Feb 15 13:40:17 2019 -0500 + + Reduce global checkpoint sync interval in disruption tests (#38931) + + We verify seq_no_stats is aligned between copies at the end of some + disruption tests. Sometimes, the assertion `assertSeqNos` is tripped due + to a lagged global checkpoint on replicas. The global checkpoint on + replicas is lagged because we sync the global checkpoint 30 seconds (by + default) after the last replication operation. This change reduces the + global checkpoint sync-internal to 1s in the disruption tests. + + Closes #38318 + Closes #36789 + +commit a67b9f6d1f9bbdcc2eaf1e6631658900614fa3cc +Author: Nhat Nguyen +Date: Fri Feb 15 12:48:56 2019 -0500 + + Relax testStressMaybeFlushOrRollTranslogGeneration (#38918) + + The predicate shouldPeriodicallyFlush is determined by the uncommitted + translog size and the local checkpoint. The uncommitted translog size + depends on the local checkpoint. The condition shouldPeriodicallyFlush + can be true twice in in the test in the following scenario: + + 1. Index doc-0 and advances the local checkpoint to 0, the condition + shouldPeriodicallyFlush remains false. + + 2. Index doc-1 and add it to translog, but the local checkpoint is not + advanced yet (still 0). The condition shouldPeriodicallyFlush becomes + true because the uncommitted translog size is 216bytes (2ops + gen-1 + + gen-2) > 180bytes and the translog generation of the new index commit + would advance from 1 to 2. + + > [2019-02-13T23:33:58,257][TRACE][o.e.i.e.Engine ] [node_s_0] + > [test][0] committing writer with commit data [{local_checkpoint=0, + > max_unsafe_auto_id_timestamp=-1, translog_uuid=fFp1Yqd4QiqKDD4ZrC8F-g, + > min_retained_seq_no=0, history_uuid=cn31yrwVQk-Vs7qcg4bi_Q, + > retention_leases=primary_term:1;version:0;, translog_generation=2, + > max_seq_no=1}] + + 1. The shouldPeriodicallyFlush becomes true again after the local + checkpoint is advanced to 1 because the uncommitted translog size is + 216bytes (2ops + gen-2 + gen-3) > 180bytes and the translog generation + of the new index commit would advance from 2 to 4. + + > [2019-02-13T23:33:58,264][TRACE][o.e.i.e.Engine ] [node_s_0] + > [test][0] committing writer with commit data [{local_checkpoint=1, + > max_unsafe_auto_id_timestamp=-1, translog_uuid=fFp1Yqd4QiqKDD4ZrC8F-g, + > min_retained_seq_no=0, history_uuid=cn31yrwVQk-Vs7qcg4bi_Q, + > retention_leases=primary_term:1;version:0;, translog_generation=4, + > max_seq_no=1}] + + We need to relax the assertion in this test to cover this situation. + + Closes #31629 + +commit 238425e5e77303d4206cbcb03789b09e3b0956bb +Author: Armin Braun +Date: Fri Feb 8 15:59:19 2019 +0100 + + Fix Issue with Concurrent Snapshot Init + Delete (#38518) + + * Fix Issue with Concurrent Snapshot Init + Delete by ensuring that we're not finalizing a snapshot in the repository while it is initializing on another thread + + * Closes #38489 + +commit 92756288b43f0ff262a37a00f10a162b983d50d7 +Author: Tal Levy +Date: Fri Feb 15 16:26:41 2019 -0800 + + relax ML Info Docs expected response (#38993) + + the get-ml-info API documentation tested that the + response show that ML's `upgrade_mode` was false. + For reasons that may be true due to other tests running in + parallel or not cleaning themselves up, this may not be + guaranteed. Since the actual value here is not of importance, + this commit relaxes the requirement that upgrade_mode be + static. + +commit 9fbfe77bb0afe0e6142d4d98bc368980373d18ff +Author: Guilherme Ferreira +Date: Fri Feb 15 21:19:23 2019 +0100 + + Fix typo in Index API doc (#38961) + +commit 63bfaac16db1d9a8b189b239e1a9c774d07b3b46 +Author: Mark Vieira +Date: Fri Feb 15 13:31:05 2019 -0800 + + Improve testcluster distribution artifact handling (#38933) (#38981) + + This commit moves validation logic for ensuring our testclusters + configuration doesn't contain unexpected artifacts into the plugin + itself. This change allows us to remove the custom copy task + implementation altogether. + + Additionally, the error message has been improved to display component + ids in addition to the artifacts to make it easier to figure out what + actual dependency is at fault. + +commit 58551198d5bef3f5b4916bcb97da3777e1762159 +Author: Jason Tedor +Date: Fri Feb 15 15:48:56 2019 -0500 + + Address some CCR REST test case flakiness (#38975) + + The CCR REST tests that rely on these assertions are flaky. They are + flaky since the introduction of recovery from the remote. + + The underlying problem is this: these tests are making assertions about + the number of operations read by the shard following task. However, with + recovery from remote, we no longer have guarantees that the assumptions + these tests were relying on hold. Namely, these tests were assuming that + the only way that a document could land in the follower index is via the + shard following task. With recovery from remote, there is another way, + which is via the files that are copied over during the recovery + phase. Most of the time this will not be a problem because with the + small number of documents that we are indexing in these tests, it is + usally not the case that a flush would occur and so there would not be + any documents in the files copied over. However, a flush can occur any + time at which point all of the indexed documents could end up in a safe + commit and copied over during recovery from remote. This commit modifies + these assertions to ones that are not prone to this issue, yet still + validate the health of the follower shard. + +commit dc0e6570919128e29a4007cb89da6dae476728dc +Author: Darren Meiss +Date: Fri Feb 15 15:51:33 2019 -0500 + + Edits to text & formatting in Term Suggester doc (#38963) + +commit 56997bf53da504931557341c44ae1a5179b05414 +Author: Darren Meiss +Date: Fri Feb 15 14:39:46 2019 -0500 + + Edits to text in Completion Suggester doc (#38980) + +commit c5dce4266731ac8dadc4cfd90721f35b23407244 +Author: Costin Leau +Date: Fri Feb 15 22:12:08 2019 +0200 + + SQL: doc polishing + +commit 79bc6aba79fc2d98f38e8f4d4695a93346b1fd40 +Author: Costin Leau +Date: Fri Feb 15 21:15:40 2019 +0200 + + SQL: Polish the rest chapter (#38971) + + Organize the text a bit and add tip on triple quotes in Kibana Console + +commit e78f70e5d8d099b55496dd40fd4655af194635d5 +Author: lcawl +Date: Fri Feb 15 11:09:49 2019 -0800 + + [DOCS] Fixes broken formatting + +commit 9f6c77fad412895a61d5393fb8ddddee6cac1d8b +Author: Christoph Büscher +Date: Fri Feb 15 20:12:26 2019 +0100 + + Fix FullClusterRestartIT#testSnapshotRestore (#38795) + + This test failed on 7.1 when running full cluster restart tests against pre-7.0 + clusters (e.g. 6.6 clusters). The fixes the expected type in the + templates after the cluster restart. + +commit 339a15bb0959960c0bbd49fb357a706860b215b7 +Author: Lisa Cawley +Date: Fri Feb 15 09:37:34 2019 -0800 + + [DOCS] Edits warning in put watch API (#38582) + +commit d300048cd5f1360ac2787703c401936fbb9ce186 +Author: Lisa Cawley +Date: Fri Feb 15 09:29:45 2019 -0800 + + [DOCS] Updates methods for upgrading machine learning (#38876) (#38967) + +commit 03b67b3ee10b0e9dbe731e2a85ab0b2dbdc11b2c +Author: Martijn van Groningen +Date: Fri Feb 15 16:14:20 2019 +0100 + + Introduced class reuses follow parameter code between ShardFollowTasks (#38910) + + and AutoFollowPattern classes. + + The ImmutableFollowParameters is like the already existing FollowParameters, + but all of its fields are final. + +commit 176013e23ce9c429c612e0fadc1d692de6443096 +Author: Alan Woodward +Date: Fri Feb 15 15:42:04 2019 +0000 + + Avoid double term construction in DfsPhase (#38716) + + DfsPhase captures terms used for scoring a query in order to build global term statistics across + multiple shards for more accurate scoring. It currently does this by building the query's `Weight` + and calling `extractTerms` on it to collect terms, and then calling `IndexSearcher.termStatistics()` + for each collected term. This duplicates work, however, as the various `Weight` implementations + will already have collected these statistics at construction time. + + This commit replaces this round-about way of collecting stats, instead using a delegating + IndexSearcher that collects the term contexts and statistics when `IndexSearcher.termStatistics()` + is called from the Weight. + + It also fixes a bug when using rescorers, where a `QueryRescorer` would calculate distributed term + statistics, but ignore field statistics. `Rescorer.extractTerms` has been removed, and replaced with + a new method on `RescoreContext` that returns any queries used by the rescore implementation. + The delegating IndexSearcher then collects term contexts and statistics in the same way described + above for each Query. + +commit 27cf7e27e769fcff2895dd42feff855182aa6c8e +Author: Hannes Van De Vreken +Date: Fri Feb 15 16:18:12 2019 +0100 + + Fix typo in DateRange docs (yyy → yyyy) (#38883) + +commit b19b778cbbb0fd8d66b9476a3595fab307fcc198 +Author: iverase +Date: Fri Feb 15 16:02:46 2019 +0100 + + [CI] Muting method testFollowIndex in IndexFollowingIT + + Relates to #38949 + +commit 78541dfb2a9c6c16a1be5d0944c8925e2aa43197 +Author: David Pilato +Date: Fri Feb 15 15:55:41 2019 +0100 + + Update Lucene snapshot repo for 7.0.0-beta1 (#38946) + + This commit updates the documentation for the Lucene snapshot repo. + +commit fcc7f553f54f5e886d4387a07627bdfc739020fa +Author: Daniel Mitterdorfer +Date: Fri Feb 15 15:34:40 2019 +0100 + + Also mmap cfs files for hybridfs (#38940) (#38947) + + With this commit we add the `.cfs` file extension to the list of file + types that are memory-mapped by hybridfs. `.cfs` files combine all files + of a Lucene segment into a single file in order to save file handles. As + this strategy is only used for "small" segments (less than 10% of the + shard size), it is benefical to memory-map them instead of accessing + them via NIO. + + Relates #36668 + +commit 9357c172881ba311247ab90c72839a5da56276bc +Author: Costin Leau +Date: Fri Feb 15 15:24:03 2019 +0200 + + SQL: Doc on syntax (identifiers in particular) (#38662) + + Add section on syntax, identifiers and literals and on single vs double quotes. + + (cherry picked from commit aafdb598082e451f36294bd174d0887a276d8c7f) + +commit eb7da4b90db5cf246c8be49fad1eb040ccc290d4 +Author: Alpar Torok +Date: Fri Feb 15 15:03:29 2019 +0200 + + Upgrade to Gradle 5.2.1 (#38880) + +commit 6181d2f4f422c5eff4bdd595ee4b6011384c8c5b +Author: Marios Trivyzas +Date: Fri Feb 15 15:06:49 2019 +0200 + + Build: Fix issue with test status logging (#38799) (#38943) + + Handle the case of `Description` being null which is a valid case as + described in the `HeartBeatEvent`'s javadoc, which previously resulted + in exceptions that "pollute" the build output. + + Follows: #28563 + Backport: #38799 + +commit 36c274867eac90a063a18e7586aad580087ce59c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Feb 15 23:01:35 2019 +1100 + + Fix intermittent failure in ApiKeyIntegTests (#38627) (#38935) + + Few tests failed intermittently and most of the + times due to invalidated or expired keys that were + deleted were still reported in search results. + This commit removes the test and adds enhancements + to other tests testing different scenario's. + + When ExpiredApiKeysRemover is triggered, the tests + did not await its termination thereby sometimes + the results would be wrong for a search operation. + + DELETE_INTERVAL setting has been further reduced to + 100ms so we can trigger ExpiredApiKeysRemover faster. + + Closes #38408 + +commit 60cc04ed13520a130ef2647717a540231e5c1eb4 +Author: Martijn van Groningen +Date: Fri Feb 15 10:24:50 2019 +0100 + + Migrate muted auto follow rolling upgrade test and unmute this test (#38900) + + The rest of `CCRIT` is now no longer relevant, because the remaining + test tests the same of the index following test in the rolling upgrade + multi cluster module. + + Added `tests.upgrade_from_version` version to test. It is not needed + in this branch, but is in 6.7 branch. + + Closes #37231 + +commit 578514e89266bd8f7d032b4a3e62e8a2bbe863a3 +Author: David Turner +Date: Fri Feb 15 08:43:24 2019 +0000 + + Recover peers from translog, ignoring soft deletes (#38904) + + Today if soft deletes are enabled then we read the operations needed for peer + recovery from Lucene. However we do not currently make any attempt to retain + history in Lucene specifically for peer recoveries so we may discard it and + fall back to a more expensive file-based recovery. Yet we still retain + sufficient history in the translog to perform an operations-based peer + recovery. + + In the long run we would like to fix this by retaining more history in Lucene, + possibly using shard history retention leases (#37165). For now, however, this + commit reverts to performing peer recoveries using the history retained in the + translog regardless of whether soft deletes are enabled or not. + +commit a211e51343999cd132377682f7febb54be2eb22f +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Feb 15 10:10:27 2019 +0100 + + ShardBulkAction ignore primary response on primary (#38901) + + Previously, if a version conflict occurred and a previous primary + response was present, the original primary response would be used both + for sending to replica and back to client. This was made in the past as an + attempt to fix issues with conflicts after relocations where a bulk request + would experience a closed shard half way through and thus have to retry + on the new primary. It could then fail on its own update. + + With sequence numbers, this leads to an issue, since if a primary is + demoted (network partitions), it will send along the original response + in the request. In case of a conflict on the new primary, the old + response is sent to the replica. That data could be stale, leading to + inconsistency between primary and replica. + + Relocations now do an explicit hand-off from old to new primary and + ensures that no operations are active while doing this. Above is thus no + longer necessary. This change removes the special handling of conflicts + and ignores primary responses when executing shard bulk requests on the + primary. + +commit d55e52223f77e0aeb96c749a481e843e5122f941 +Author: Yannick Welsch +Date: Fri Feb 15 07:51:29 2019 +0100 + + Smarter CCR concurrent file chunk fetching (#38841) + + The previous logic for concurrent file chunk fetching did not allow for multiple chunks from the same + file to be fetched in parallel. The parallelism only allowed to fetch chunks from different files in + parallel. This required complex logic on the follower to be aware from which file it was already + fetching information, in order to ensure that chunks for the same file would be fetched in sequential + order. During benchmarking, this exhibited throughput issues when recovery came towards the end, + where it would only be sequentially fetching chunks for the same largest segment file, with + throughput considerably going down in a high-latency network as there was no parallelism anymore. + + The new logic here follows the peer recovery model more closely, and sends multiple requests for + the same file in parallel, and then reorders the results as necessary. Benchmarks show that this + leads to better overall throughput and the implementation is also simpler. + +commit 1f74ba2d332976fa95a51d5476d09bdf8deb6914 +Author: Shaunak Kashyap +Date: Thu Feb 14 16:09:52 2019 -0800 + + [Monitoring] Remove `include_type_name` parameter from GET _template request (#38925) + + Backport of #38818 to `7.x`. Original description: + + The HTTP exporter code in the Monitoring plugin makes `GET _template` requests to check for existence of templates. These requests don't need to pass the `include_type_name` query parameter so this PR removes it from the request. This should remove the following deprecation log entries on the Monitoring cluster in 7.0.0 onwards: + + ``` + [types removal] Specifying include_type_name in get index template requests is deprecated. + ``` + +commit 5d06226507943bff6023e89ee5adf0e1eae69828 +Author: Jay Modi +Date: Thu Feb 14 16:28:52 2019 -0700 + + Fix writing of SecurityFeatureSetUsage to pre-7.1 (#38922) + + This change makes the writing of new usage data conditional based on + the version that is being written to. A test has also been added to + ensure serialization works as expected to an older version. + + Relates #38687, #38917 + +commit 7d449c5f658ec4367ad683318c7e3b2e0ba947a3 +Author: Lee Hinman +Date: Thu Feb 14 13:46:17 2019 -0700 + + Check that delete index request succeeded in test teardown (#38903) (#38913) + + Backport of #38903 + + When tearing down from `ESSingleNodeTestCase` we perform a delete on "*" + indices, it some cases, however, those indices are not fully deleted. Rather + than have a failure occur later down the change (see: + https://github.com/elastic/elasticsearch/issues/30290#issuecomment-463589008 ) + the failure should occurr immediately so it can be diagnosed more easily. + +commit 00cb8d0be84e2d22d06f62d3cb181e7fd9b484ea +Author: Jason Tedor +Date: Thu Feb 14 12:37:59 2019 -0500 + + Mark coordinator test as awaits fix + + This test is failing frequently so this commit mutes it. + + Relates #38867 + +commit 0c733c04be90654b34cfa2aa6bf7fa7bdbb94c47 +Author: Lee Hinman +Date: Thu Feb 14 09:30:40 2019 -0700 + + Remove immediate operation retry after mapping update (#38873) + + Prior to this commit, when an indexing operation resulted in an + `Engine.Result.Type.MAPPING_UPDATE_REQUIRED`, TransportShardBulkAction + immediately retries the indexing operation to see if it succeeds. In the event + that it succeeds the context does not wait until the mapping update has + propagated through the cluster state before finishing the indexing. + + In some of our tests we rely on mappings being available as soon as they've been + introduced in a document that indexed correctly. By removing the immediate retry + we always wait for this to be the case. + + Resolves #38428 + Supercedes #38579 + Relates to #38711 + +commit d9c255dbbf393d3d358f5900b3072730dad9f911 +Author: debadair +Date: Thu Feb 14 07:42:26 2019 -0800 + + [DOCS] Added include and reference to beta1 RNs (#38905) + +commit 52a4ca5962281298fcddf98468abd7a65cd39c08 +Author: Christoph Büscher +Date: Thu Feb 14 16:31:59 2019 +0100 + + Remove mentioning of types from bulk API docs (#38896) + + The docs on master still mention types in the context of conflicts with + documents and also mentions the deprecated endpoint including types. + +commit 99551001cd380a73d8b1477ffed0bc0dd4788cba +Author: Alpar Torok +Date: Thu Feb 14 17:19:28 2019 +0200 + + Skip BWC tests in checkPart1 and checkPart2 (#38730) + + Don't run bwc tests for check part 1 and 2 + +commit 6c5cec4ff4054820d4a984712ea6153158411359 +Author: Christoph Büscher +Date: Thu Feb 14 16:14:48 2019 +0100 + + Enable silent FollowersCheckerTest (#38851) + + One of the test methods wasn't run because it was private. Making this method + public and fixing some issues around mocking the threadpool that otherwise would + lead to an NPE. + +commit cf0bdf3b28c0769d8241ecab3a4caa7011d8138d +Author: Lee Hinman +Date: Thu Feb 14 08:13:05 2019 -0700 + + Update TESTING.asciidoc with platform specific instructions (#38802) + + This adds the instructions for building a platform-specific distribution. + +commit 8bb2a2a405485291381f9a0d6819a85564f439f5 +Author: taku333 <19281989thezuka@gmail.com> +Date: Thu Feb 14 21:20:07 2019 +0900 + + SQL: change JDBC setup URL in the documentation (#38564) + + (cherry picked from commit 103786ea27da72b2fccd3cf511b3143dae0fc530) + +commit e59b7b696a54a72ea9c3151210cdc8da7c36f46d +Author: Jay Modi +Date: Thu Feb 14 07:34:29 2019 -0700 + + Use consistent view of realms for authentication (#38815) + + This change updates the authentication service to use a consistent view + of the realms based on the license state at the start of + authentication. Without this, the license can change during + authentication of a request and it will result in a failure if the + realm that extracted the token is no longer in the realm list. This + manifests in some tests as an authentication failure that should never + really happen; one example would be the test framework's transport + client user should always have a succesful authentication but in the + LicensingTests this can fail and will show up as a + NoNodeAvailableException. + + Additionally, the licensing tests have been updated to ensure that + there is consistency when changing the license. The license is changed + by modifying the internal xpack license state on each node, which has + no protection against be changed by some pending cluster action. The + methods to disable and enable now ensure we have a green cluster and + that the cluster is consistent before returning. + + Closes #30301 + +commit 6243a9797fad236e1a313d04fcd18f79c8eabf13 +Author: Albert Zaharovits +Date: Thu Feb 14 12:50:31 2019 +0200 + + _cat/indices with Security, hide names when wildcard (#38824) + + This changes the output of the `_cat/indices` API with `Security` enabled. + + It is possible to only display the index name (and possibly the index + health, depending on the request options) but not its stats (doc count, merges, + size, etc). This is the case for closed indices which have index metadata in the + cluster state but no associated shards, hence no shard stats. + However, when `Security` is enabled, and the request contains wildcards, + **open** indices without stats are a common occurrence. This is because the + index names in the response table are picked up directly from the cluster state + which is not filtered by `Security`'s _indexNameExpressionResolver_, unlike the + stats data which is populated by the indices stats API which does go through the + index name resolver. + This is a bug, because it is circumventing `Security`'s function to hide + unauthorized indices. + + This has been fixed by displaying the index names as they are resolved by the indices + stats API. The outputs of these two APIs is now very similar: same index names, + similar data but different format. + + Closes #37190 + +commit 7d78f4641b22884d70aca968b56f30e15c65abe8 +Author: Andrei Stefan +Date: Thu Feb 14 12:42:18 2019 +0200 + + SQL: fall back to using the field name for column label (#38842) + + (cherry picked from commit 0567bf24957be477e7649cff94872b0e7dc4d284) + +commit 335cf91bb9929866b8e94f06bc0c961a2846bf01 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Feb 14 23:08:52 2019 +1100 + + Add enabled status for token and api key service (#38687) (#38882) + + Right now there is no way to determine whether the + token service or API key service is enabled or not. + This commit adds support for the enabled status of + token and API key service to the security feature set + usage API `/_xpack/usage`. + + Closes #38535 + +commit 96e7d71948f267a0b95b3fc45a1e6df87b48fb3c +Author: Martijn van Groningen +Date: Wed Feb 13 11:16:44 2019 +0100 + + Handle the fact that `ShardStats` instance may have no commit or seqno stats (#38782) + + The should fix the following NPE: + + ``` + [2019-02-11T23:27:48,452][WARN ][o.e.p.PersistentTasksNodeService] [node_s_0] task kD8YzUhHTK6uKNBNQI-1ZQ-0 failed with an exception + 1> java.lang.NullPointerException: null + 1> at org.elasticsearch.xpack.ccr.action.ShardFollowTasksExecutor.lambda$fetchFollowerShardInfo$7(ShardFollowTasksExecutor.java:305) ~[main/:?] + 1> at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:61) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:68) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:64) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onCompletion(TransportBroadcastByNodeAction.java:383) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onNodeResponse(TransportBroadcastByNodeAction.java:352) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:324) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:314) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1108) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1189) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1169) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:54) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:417) [elasticsearch-8.0.0-SNAP + SHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:391) [elasticsearch-8.0.0-SNAP + SHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] + 1> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_202] + 1> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_202] + 1> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202] + ``` + + Relates to #38779 + +commit 6ea483a66309a646812f6da778d24b18db90f3da +Author: David Roberts +Date: Thu Feb 14 11:46:22 2019 +0000 + + Mute DedicatedClusterSnapshotRestoreIT testRestoreShrinkIndex + + Due to https://github.com/elastic/elasticsearch/issues/38845 + +commit 21f76aba28c2473d75457326d8603874e0eaa812 +Author: Dimitris Athanasiou +Date: Thu Feb 14 12:15:00 2019 +0200 + + [ML] Extract base class for integ tests with native processes (#38850) (#38860) + +commit 745611701995c41a4401aac60fb11dfdbf5b043d +Author: Luca Cavanna +Date: Tue Feb 12 13:31:06 2019 +0100 + + [TEST] address testCollectNodes rare failure (#38559) + + #37767 changed the expected exception for "no such cluster" error from + `IllegalStateException` to a dedicated `NoSuchRemoteClusterException`. + An assertion in `testCollectNodes` needs to be updated accordingly. + +commit 8e5e48319e04e359d8881fcf1400aa66c9d09b7b +Author: Alexander Reelsen +Date: Thu Feb 14 10:18:12 2019 +0100 + + Add documentation about breaking java time changes (#38886) + + In addition remove joda time mentions across the docs, make + sure links are updated to java time javadocs. + + Forward port of #38720 + +commit 88489a3f3a432e0102309d12a68ff122c1e6a7f8 +Author: Martijn van Groningen +Date: Thu Feb 14 08:12:14 2019 +0100 + + Backport rolling upgrade multi cluster module (#38859) + + * Add rolling upgrade multi cluster test module (#38277) + + This test starts 2 clusters, each with 3 nodes. + First the leader cluster is started and tests are run against it and + then the follower cluster is started and tests execute against this two cluster. + + Then the follower cluster is upgraded, one node at a time. + After that the leader cluster is upgraded, one node at a time. + Every time a node is upgraded tests are ran while both clusters are online. + (and either leader cluster has mixed node versions or the follower cluster) + + This commit only tests CCR index following, but could be used for CCS tests as well. + In particular for CCR, unidirectional index following is tested during a rolling upgrade. + During the test several indices are created and followed in the leader cluster before or + while the follower cluster is being upgraded. + + This tests also verifies that attempting to follow an index in the upgraded cluster + from the not upgraded cluster fails. After both clusters are upgraded following the + index that previously failed should succeed. + + Relates to #37231 and #38037 + + * Filter out upgraded version index settings when starting index following (#38838) + + The `index.version.upgraded` and `index.version.upgraded_string` are likely + to be different between leader and follower index. In the event that + a follower index gets restored on a upgraded node while the leader index + is still on non-upgraded nodes. + + Closes #38835 + +commit 60c1dcde88681a46e221081430bb3f6346c9cc46 +Author: Lee Hinman +Date: Wed Feb 13 16:13:14 2019 -0700 + + Only flush Watcher's bulk processor if Watcher is enabled (#38803) + + When shutting down Watcher, the `bulkProcessor` is null if watcher has been + disabled in the configuration. This protects the flush and close calls with a + check for watcher enabled to avoid a NullPointerException + + Resolves #38798 + +commit 4ad4bc7f5fbf68b59bc20f953f57e798526f4513 +Author: Julie Tibshirani +Date: Wed Feb 13 14:43:35 2019 -0800 + + Update the removal of types docs with the new 6.7 behavior. (#38869) + + Follow-up to #38825, where we made a tweak to the deprecation behavior. + +commit 5d22e459905470dc5409f059920504f02a44ea5f +Author: Nhat Nguyen +Date: Mon Feb 11 20:36:27 2019 -0500 + + Copy retention leases when trim unsafe commits (#37995) + + When a primary shard is recovered from its store, we trim the last + commit (when it's unsafe). If that primary crashes before the recovery + completes, we will lose the committed retention leases because they are + baked in the last commit. With this change, we copy the retention leases + from the last commit to the safe commit when trimming unsafe commits. + + Relates #37165 + +commit ec08581319fa1ee2f2185a68bb0503677f5b6f5f +Author: Tim Brooks +Date: Wed Feb 13 07:18:24 2019 -0700 + + Improve CcrRepositoryIT mappings tests (#38817) + + Currently we index documents concurrently to attempt to ensure that we + update mappings during the restore process. However, this does not + actually test that the mapping will be correct and is dangerous as it + can lead to a misalignment between the max sequence number and the local + checkpoint. If these are not aligned, peer recovery cannot be completed + without initiating following which this test does not do. That causes + teardown assertions to fail. + + This commit removes the concurrent indexing and flushes after the + documents are indexed. Additionally it modifies the mapping specific + test to ensure that there is a mapping update when the restore session + is initiated. This mapping update is picked up at the end of the restore + by the follower. + +commit e769cb4efde3a2c06ea9d4c7a52f42113599babd +Author: Julie Tibshirani +Date: Wed Feb 13 10:40:18 2019 -0800 + + Perform precise check for types warnings in cluster restart tests. (#37944) + + Instead of using `WarningsHandler.PERMISSIVE`, we only match warnings + that are due to types removal. + + This PR also renames `allowTypeRemovalWarnings` to `allowTypesRemovalWarnings`. + + Relates to #37920. + +commit 062eea8fccc80c1a1d678ff3bc5ddb14c6e32e02 +Author: Jason Tedor +Date: Wed Feb 13 14:04:11 2019 -0500 + + Fix excessive increments in soft delete policy (#38813) + + In this case, we were incrementing the policy too much. This means on + every iteration we actually keep increasing the minimum retained + sequence number, even with leases in place. It was a bug from when the + soft deletes policy had retention leases incorporated into it. This + commit fixes this bug by ensuring we only increment in the proper + places, and adds careful tests for the various situations. + +commit d2ac05e249b31d56c7f2c2c97735bee29526cd51 +Author: Benjamin Trent +Date: Wed Feb 13 10:58:55 2019 -0600 + + ML allow aliased .ml-anomalies* index on PUT Job (#38821) (#38847) + +commit 46bb663a09c9c02dcbec30b2763358adf1963eea +Author: Jake Landis +Date: Wed Feb 13 10:28:01 2019 -0600 + + Make 7.x like 6.7 user agent ecs, but default to true (#38828) + + Forward port of https://github.com/elastic/elasticsearch/pull/38757 + + This change reverts the initial 7.0 commits and replaces them + with the 6.7 variant that still allows for the ecs flag. + This commit differs from the 6.7 variants in that ecs flag will + now default to true. + + 6.7: `ecs` : default `false` + 7.x: `ecs` : default `true` + 8.0: no option, but behaves as `true` + + * Revert "Ingest node - user agent, move device to an object (#38115)" + This reverts commit 5b008a34aa3c07e37b12b415d3c22a44da491329. + + * Revert "Add ECS schema for user-agent ingest processor (#37727) (#37984)" + This reverts commit cac6b8e06f051d68919faf6081f1c87fa5b6757d. + + * cherry-pick 5dfe1935345da3799931fd4a3ebe0b6aa9c17f57 + Add ECS schema for user-agent ingest processor (#37727) + + * cherry-pick ec8ddc890a34853ee8db6af66f608b0ad0cd1099 + Ingest node - user agent, move device to an object (#38115) (#38121) + + * cherry-pick f63cbdb9b426ba24ee4d987ca767ca05a22f2fbb (with manual merge fixes) + Dep. check for ECS changes to User Agent processor (#38362) + + * make true the default for the ecs option, and update 7.0 references and tests + +commit 7404882105d707c10aefa803e293174012750afe +Author: Przemyslaw Gomulka +Date: Wed Feb 13 13:34:33 2019 +0100 + + Fix line separators in JSON logging tests backport#38771 #38834 + + The hardcoded '\n' in string will not work in Windows where there is a + different line separator. A System.lineSeparator should be used to make + it work on all platforms + closes #38705 + backport #38771 + +commit 34b6383c47c377c1ef8f7473bfadfdbe2e889e45 +Author: Darren Meiss +Date: Wed Feb 13 04:07:16 2019 -0500 + + Edits to text of Profile API documentation (#38742) + + Minor edits of text. + +commit 251ced79eee3601d01ad6f32a35c4d1c89b9587e +Author: Jim Ferenczi +Date: Wed Feb 13 09:52:58 2019 +0100 + + Add release notes for 7.0.0-beta1 (#38729) + + Add release notes for 7.0.0-beta1 + +commit 470ac0ec959542fa82aee2bc09f36bf597b0d9c1 +Author: Dimitrios Liappis +Date: Wed Feb 13 09:56:47 2019 +0200 + + Don't fail init script if `/proc/.../max_map_count` absent (#38789) + + Currently init scripts fail when `/proc/sys/vm/max_map_count` is not present + with `-bash: [: too many arguments`. + + Fix conditional logic to avoid trying to set the `max_map_count` sysctl if not + present. + + Backport of: #35933 + Relates: #27236 + +commit 542ee5f46a74f413f6c07bac6212682e54d91341 +Author: Przemyslaw Gomulka +Date: Wed Feb 13 08:33:53 2019 +0100 + + Format Watcher.status.lastChecked and lastMetCondition (#38788) backport#38626 + + Change the formatting for Watcher.status.lastCheck and lastMetCondition + to be the same as Watcher.status.state.timestamp. These should all have + only millisecond precision + closes #38619 + backport #38626 + +commit a9178b32394fec40f3909fcd2f4f623deb1c8239 +Author: Shaunak Kashyap +Date: Tue Feb 12 19:54:36 2019 -0800 + + Remove _type term filters from cluster alert watches (#38819) (#38826) + + Backport of https://github.com/elastic/elasticsearch/pull/38819. Original message: + + This PR removes usages of the `_type` field in `_search` requests issued from Monitoring code. + +commit 57f69082fd316e9f23896ef320a9577b13f6e384 +Author: Zachary Tong +Date: Tue Feb 12 13:11:52 2019 -0500 + + Disable cache on QueryProfilerIT (#38748) + + - Disables the request cache on the test, to prevent cached + values from potentially interfering with test results + - Changes the test to execute a single query, in hopes of making + failures more reproducible + Backport of #38583 + +commit a3f39741bee38c88cf82b7ea60aa43f3157b2ffd +Author: Nhat Nguyen +Date: Tue Feb 12 11:40:58 2019 -0500 + + Adjust log and unmute testFailOverOnFollower (#38762) + + There were two documents (seq=2 and seq=103) missing on the follower in + one of the failures of `testFailOverOnFollower`. I spent several hours + on that failure but could not figure out the reason. I adjust log and + unmute this test so we can collect more information. + + Relates #38633 + +commit 4a5070dcfba02612e35b34c4b22a7cf88ed65edf +Author: Nhat Nguyen +Date: Mon Feb 11 13:12:05 2019 -0500 + + Use current term in initial leases in engine test (#38285) + + We need to use the current primary term instead of 1L for the initial + retention leases; otherwise, the primary term of the committed + retention leases won't match the current primary term if the + retention leases never gets updated. + +commit eca54045729d7b660d7eda1f4abf083a6dd1df1d +Author: Nhat Nguyen +Date: Tue Feb 12 11:06:22 2019 -0500 + + Fix synchronization in LocalCheckpointTracker#contains (#38755) + + We are accessing the `CountedBitSet` in `LocalCheckpointTracker#contains` + without proper synchronization. + + Relates #33871 + +commit 225ebb6935dae38f93107c93c3a3fc93ecb436af +Author: Nhat Nguyen +Date: Mon Feb 11 10:21:20 2019 -0500 + + Ensure no snapshotted commit when close engine (#38663) + + With this change, we can automatically detect an implementation + that acquires an index commit but fails to release. + +commit f04bd4a07eac3e32fcf58c9d06cd2b2a9b1bcfc2 +Author: Jay Modi +Date: Tue Feb 12 08:43:34 2019 -0700 + + Remove TLSv1.2 pinning in ssl reload tests (#38651) + + This change removes the pinning of TLSv1.2 in the + SSLConfigurationReloaderTests that had been added to workaround an + issue with the MockWebServer and Apache HttpClient when using TLSv1.3. + The way HttpClient closes the socket causes issues with the TLSv1.3 + SSLEngine implementation that causes the MockWebServer to loop + endlessly trying to send the close message back to the client. This + change wraps the created http connection in a way that allows us to + override the closing behavior of HttpClient. + + An upstream request with HttpClient has been opened at + https://issues.apache.org/jira/browse/HTTPCORE-571 to see if the method + of closing can be special cased for SSLSocket instances. + + This is caused by a JDK bug, JDK-8214418 which is fixed by + https://hg.openjdk.java.net/jdk/jdk12/rev/5022a4915fe9. + + Relates #38646 + +commit 6eec065353cbfd0766ce0cf966f6552b057bb3e1 +Author: Mayya Sharipova +Date: Tue Feb 12 11:09:15 2019 -0500 + + Describe what _source.includes/excludes do (#38319) (#38794) + +commit 40d5beaf416a3f461a01806f51ed248436bc0838 +Author: Martijn van Groningen +Date: Tue Feb 12 16:54:34 2019 +0100 + + muted test + + Relates to #38779 + +commit 032bcf99d62dac0d219f43cd7350cde314ef51fd +Author: Marios Trivyzas +Date: Tue Feb 12 16:50:21 2019 +0200 + + SQL: Implement `::` cast operator (#38774) + + `::` is a simplified altenative syntax to + `CAST( AS which exists in PostgreSQL and + provides an improved user experience and possibly more compact + SQL queries. + + Fixes: #38717 + +commit 12eac6ad4b4d2a85dfc5e2847c492d80e81e4c02 +Author: Alpar Torok +Date: Tue Feb 12 15:55:08 2019 +0200 + + Mute failing test (#38781) + + Tracking #38711 + +commit 51d6b9ab316906abc80950c7f4a980ff2bf1a6cb +Author: Tanguy Leroux +Date: Tue Feb 12 14:02:56 2019 +0100 + + Fix CloseWhileRelocatingShardsIT (#38728) + +commit bbc9aa997909a70b5a10df0fe23a2812edb03a83 +Author: Jason Tedor +Date: Tue Feb 12 07:37:12 2019 -0500 + + Introduce retention lease actions (#38756) + + This commit introduces actions for some common retention lease + operations that clients need to be able to perform remotely. These + actions include add/renew/remove. + +commit 745d0c1bbabf0d246a9560b38effe418c59dc9f6 +Author: Alpar Torok +Date: Tue Feb 12 14:19:27 2019 +0200 + + Add packaging tests for Fedora 29, remove 27 (#38726) + +commit 085b6b5f899139e48e0428d24be7290987670d00 +Author: Alpar Torok +Date: Tue Feb 12 14:18:52 2019 +0200 + + Fix failing bwc test against 6.3 (#38770) + +commit 7e178aa4a7f9836adaa27ee7b76348127aeed1c3 +Author: Przemyslaw Gomulka +Date: Tue Feb 12 13:05:44 2019 +0100 + + Enable IndexActionTests and WatcherIndexingListenerTests Backport #38738 + + fix tests to use clock in milliseconds precision in watcher code + make sure the date comparison in string format is using same formatters + some of the code was modified in #38514 possibly because of merge conflicts + + closes #38581 + Backport #38738 + +commit 90fff54954c0413003f0d3dba3752b3c7b674f55 +Author: Luca Cavanna +Date: Mon Feb 11 16:00:54 2019 +0100 + + Tie break on cluster alias when merging shard search failures (#38715) + + A recent test failure triggered an edge case scenario where failures may be coming back with the same shard id, yet from different clusters. + This commit adapts the failures comparator to take the cluster alias into account when merging failures as part of CCS requests execution. + Also the corresponding test has been split in two: with and without + search shard target set to the failure. + + Closes #38672 + +commit 6ae7915b9ddf93d839aca0de75163a624819428f +Author: Alexander Reelsen +Date: Tue Feb 12 10:39:18 2019 +0100 + + Fix exporter tests to have reasonable dates (#38436) + + The java time formatter used in the exporter adds a plus sign to the + year, if a year with more than five digits is used. This changes the + creation of those timestamp to only have a date up to 9999. + + Closes #38378 + +commit 6290d59ffa840532fecd7b9491db0954c801b3f9 +Author: Martijn van Groningen +Date: Tue Feb 12 10:18:52 2019 +0100 + + Use clear cluster names in order to make debugging easier. + + Relates to #37681 + +commit c7cdd6a46a2c059318241e64e6d4afc8b36cddb0 +Author: Jason Tedor +Date: Tue Feb 12 00:31:29 2019 -0500 + + Add dedicated retention lease exceptions (#38754) + + When a retention lease already exists on an add retention lease + invocation, or a retention lease is not found on a renew retention lease + invocation today we throw an illegal argument exception. This puts a + burden on the caller to catch that specific exception and parse the + message. This commit relieves the burden from the caller by adding + dedicated exception types for these situations. + +commit b97c74bbab72731f8fa7d5e859c4ff20062d5745 +Author: Jason Tedor +Date: Mon Feb 11 21:17:23 2019 -0500 + + Enable removal of retention leases (#38751) + + This commit introduces the ability to remove retention leases. Explicit + removal will be needed to manage retention leases used to increase the + likelihood of operation-based recoveries syncing, and for consumers such + as ILM. + +commit e2f432a41330330d3a07e5ea75b3dae54880debb +Author: Nick Knize +Date: Mon Feb 11 16:26:45 2019 -0600 + + Fix the version check for LegacyGeoShapeFieldMapper (#38547) + + Change version check from 7.0 to 6.6 in BaseGeoShapeFieldMapper to correctly use LegacyGeoShapeFieldMapper for indexes created prior to 6.6. + +commit f8426d9b761d20e13992a417419307970fc497da +Author: Darren Meiss +Date: Mon Feb 11 10:48:13 2019 -0500 + + Fix typos in Field-Caps documentation (#38580) + + Fix typo in Field-Caps documentation + + Reworded because asciidoc was formatting the ellipse/space as a numbered list. + +commit 0e1b1959feaa91c9ec9a0f2128ce447686f51ff1 +Author: Mayya Sharipova +Date: Mon Feb 11 14:17:18 2019 -0500 + + Correct rebuilt persian analyzer (#38724) (#38744) + + Make substitution of \u200C with a space explicit + + The problem with this symbol `\u200C` in a test string, + that **SHOULD** be substituted with space in the rebuilt Persian analyzer, but it is not. + + Correcting this line `"mappings": [ "\\u200C=> "] <1>` to + `"mappings": [ "\\u200C=>\\u0020"] <1>` in solves the problem. + This change explicitly says to substitute ZWNJ with a space. + + Closes #38188 + +commit bafc7093269328efed354294fc5e7a34259455f6 +Author: Yannick Welsch +Date: Mon Feb 11 19:15:04 2019 +0100 + + Fix CCR concurrent file chunk fetching bug (#38736) + + Fixes a bug with concurrent file chunk fetching during recovery from remote where the wrong offset + was used. + +commit 078da6d9bdf3605db831fbcd4b470b501003cdde +Author: Nick Knize +Date: Mon Feb 11 11:57:55 2019 -0600 + + Fix GeoHash PrefixTree BWC (#38584) + + geo_shape indexes created before 6.6 use geohash string encoding as default tree parameter and quadtree encoding for 6.6 and later. This commit fixes bwc to use geohash encoding in LegacyGeoshapeFieldMapper for indexes created before 6.6. + +commit d1848b96fc5943f68cfecb1e2f1c9e02734edf06 +Author: David Roberts +Date: Mon Feb 11 17:33:20 2019 +0000 + + Fix possible assertion failure in IndicesQueryCache.close (#38731) + + The assertion that the stats2 map is empty in + IndicesQueryCache.close has been observed to + fail very occasionally in internal cluster tests. + + The likely cause is a cross-thread visibility + problem for a count variable. This change + makes that count volatile. + + Relates #37117 + Backport of #38714 + +commit dc212de822e7b3675ddb3950014829f092ff6d4e +Author: Tanguy Leroux +Date: Mon Feb 11 17:34:17 2019 +0100 + + Specialize pre-closing checks for engine implementations (#38702) (#38722) + + The Close Index API has been refactored in 6.7.0 and it now performs + pre-closing sanity checks on shards before an index is closed: the maximum + sequence number must be equals to the global checkpoint. While this is a + strong requirement for regular shards, we identified the need to relax this + check in the case of CCR following shards. + + The following shards are not in charge of managing the max sequence + number or global checkpoint, which are pulled from a leader shard. They + also fetch and process batches of operations from the leader in an unordered + way, potentially leaving gaps in the history of ops. If the following shard lags + a lot it's possible that the global checkpoint and max seq number never get + in sync, preventing the following shard to be closed and a new PUT Follow + action to be issued on this shard (which is our recommended way to + resume/restart a CCR following). + + This commit allows each Engine implementation to define the specific + verification it must perform before closing the index. In order to allow + following/frozen/closed shards to be closed whatever the max seq number + or global checkpoint are, the FollowingEngine and ReadOnlyEngine do + not perform any check before the index is closed. + + Co-authored-by: Martijn van Groningen + +commit 6443b46184314d654670f217c27cdb84e1c648f8 +Author: Luca Cavanna +Date: Mon Feb 11 15:55:46 2019 +0100 + + Clean up ShardSearchLocalRequest (#38574) + + Added a constructor accepting `StreamInput` as argument, which allowed to + make most of the instance members final as well as remove the default + constructor. + Removed a test only constructor in favour of invoking the existing + constructor that takes a `SearchRequest` as first argument. + Also removed profile members and related methods as they were all unused. + +commit 92201ef5632f8c1d3b6138613e61a68ff6eedabc +Author: Martijn van Groningen +Date: Mon Feb 11 15:35:39 2019 +0100 + + Catch AlreadyClosedException and use other IndexShard instance (#38630) + + Closes #38617 + +commit 884b5063a4ef1ed2e4c53e1c4fe9c13e7c9ddadb +Author: Alexander Reelsen +Date: Mon Feb 11 17:10:33 2019 +0300 + + Create ISO8601 joda compatible java time formatter (#38434) + + The existing formatter being used was not on par with the joda formatter + as it was missing the ability to parse a comma as a separator between + seconds and milliseconds. + + While a real iso8601 would be much more complex, this might be + sufficient for some more use-cases. + + The ingest date formatter now also uses the iso8601 formatter by + default. + + Closes #38345 + +commit e7868e92bd3da259453e6244d89b923ffc6bb6db +Author: Alexander Reelsen +Date: Mon Feb 11 16:30:48 2019 +0300 + + Restore date aggregation performance in UTC case (#38221) (#38700) + + The benchmarks showed a sharp decrease in aggregation performance for + the UTC case. + + This commit uses the same calculation as joda time, which requires no + conversion into any java time object, also, the check for an fixedoffset + has been put into the ctor to reduce the need for runtime calculations. + The same goes for the amount of the used unit in milliseconds. + + Closes #37826 + +commit bd4ca4c7028f9e77ffa7f57e62325384c73c1495 +Author: Alpar Torok +Date: Mon Feb 11 14:58:00 2019 +0200 + + Rename integTest to bwcTestSample for bwc test projects (#38433) + + * Rename integTest to bwcTestSample for bwc test projects + + This change renames the `integTest` task to `bwcTestSample` for projects + testing bwc to make it possible to run all the bwc tests that check + would run without running on bwc tests. + + This change makes it possible to add a new PR check on backports to make + sure these don't break BWC tests in master. + + * Rename task as per PR + +commit fe8bd757b2b6aba0e2026fb1465c1bfc92deb354 +Author: Luca Cavanna +Date: Mon Feb 11 13:40:42 2019 +0100 + + Look up connection using the right cluster alias when releasing contexts (#38570) + + Whenever phase failure is raised in AbstractSearchAsyncAction, we go and + release search contexts of shards that successfully returned their + results, prior to notifying the listener of the failure. In case we are + executing a CCS request, it's important to look-up the connection to + send the release context request to. + + This commit makes sure that the lookup takes the cluster alias into + account. We used to use `null` at all times instead which is not correct + and was not caught as any exception is caught without re-throwing it. + +commit b3695750bc385b2b4734f9bdabb9a43b5e67d4b4 +Author: Andrei Stefan +Date: Mon Feb 11 13:59:31 2019 +0200 + + Randomize the time zone properly for the current date test. (#38670) + + (cherry picked from commit 29abbb8a590cdf4f9e0c0b447d6694bb7223648e) + +commit ba9a4d13e185a2af2e109637e9cd3010b29f7046 +Author: Przemyslaw Gomulka +Date: Mon Feb 11 13:15:12 2019 +0100 + + mute Failing tests related to logging and joda-java migration backport(#38704)(#38710) + + the tests awaits fix from #38693 and #38705 and #38581 + +commit ab9e2f2e6997a34ee62fd1c8966e9b48dd207ef2 +Author: Przemyslaw Gomulka +Date: Mon Feb 11 11:34:25 2019 +0100 + + Move testToUtc test to DateFormattersTests #38698 Backport #38610 + + The test was relying on toString in ZonedDateTime which is different to + what is formatted by strict_date_time when milliseconds are 0 + The method is just delegating to dateFormatter, so that scenario should + be covered there. + + closes #38359 + Backport #38610 + +commit 7e105929404a62ca08602795e4d719d0b5283136 +Author: Alpar Torok +Date: Thu Feb 7 15:46:36 2019 +0200 + + fix verify version constants + +commit 73fcea4d2c7866458b6f4ddcbc0d2f673df2ae7b +Author: Alexander Reelsen +Date: Mon Feb 11 13:03:49 2019 +0300 + + Remove ticks in chain input documentation (#38109) + + The ticks created a literal string instead of actually accessing the + payload value. + +commit 8c624e5a20c80905160c4425d672f28dcd73cd9a +Author: Ioannis Kakavas +Date: Mon Feb 11 11:43:41 2019 +0200 + + Enhance parsing of StatusCode in SAML Responses (#38628) + + * Enhance parsing of StatusCode in SAML Responses + + elements in a failed response might contain two nested + elements. We currently only parse the first one in + order to create a message that we attach to the Exception we return + and log. However this is generic and only gives out informarion + about whether the SAML IDP believes it's an error with the + request or if it couldn't handle the request for other reasons. The + encapsulated StatusCode has a more interesting error message that + potentially gives out the actual error as in Invalid nameid policy, + authentication failure etc. + + This change ensures that we print that information also, and removes + Message and Details fields from the message when these are not + part of the Status element (which quite often is the case) + +commit a29bf2585ebd92b8ac299e2602efa724c265da5c +Author: Martijn van Groningen +Date: Mon Feb 11 10:53:04 2019 +0100 + + Added unit test for FollowParameters class (#38500) (#38690) + + A unit test that tests FollowParameters directly was missing. + +commit f61420140d4089d3f9ea52b950fd4a2661d8ed9e +Author: Christoph Büscher +Date: Mon Feb 11 10:18:13 2019 +0100 + + Use only default type in rank_eval API (#38530) + + Currently tests still use custom type names. In preparation for the final types + removal this change moves all of them to use the default "_doc" type in tests. + +commit 0e5a734e7e8caa72b007fd1b1dc9d74396eec50e +Author: Przemyslaw Gomulka +Date: Mon Feb 11 08:50:21 2019 +0100 + + Fix HistoryIntegrationTests timestamp comparison #38565 Backport#38505 + + When the millisecond part of a timestamp is 0 the toString + representation in java-time is omitting the millisecond part (joda was + not). The Search response is returning timestamps formatted with + WatcherDateTimeUtils, therefore comparisons of strings should be done + with the same formatter + + relates #27330 + BackPort #38505 + +commit 4625807505e8352409c255c0b1aa14b166c1bdc2 +Author: Martijn van Groningen +Date: Mon Feb 11 08:45:11 2019 +0100 + + Reuse FollowParameters' parse fields. (#38508) + +commit e213ad3e88252387d40ea57058a01616e68eebd9 +Author: Martijn van Groningen +Date: Mon Feb 11 08:23:01 2019 +0100 + + Mute test. + + Relates to #38695 + +commit fb1c9c25fa7df79aa7ff0cf3e40feb5d48a30241 +Author: Tim Vernum +Date: Mon Feb 11 17:16:55 2019 +1100 + + Add docs for new reindex.ssl settings (#38686) + + Reindex from remote now supports configurable SSL/TLS (node level) + settings. This change adds documentation relating to those settings + + Relates: #37527 + Backport of: #38486 + +commit 273edea712e480c896c42fcbaadfdbabef4a0cbb +Author: Tim Vernum +Date: Mon Feb 11 16:50:10 2019 +1100 + + Mute testExpiredApiKeysDeletedAfter1Week (#38683) + + Tracked: #38408 + +commit b8be6cb5c7ca7af0b10522426e525192b6b145dc +Author: Like +Date: Mon Feb 11 02:16:29 2019 +0800 + + Reject index.optimize_auto_generated_id setting (#28895) + + This commit rejects the index.optmize_auto_generated_id setting for + indices created on or after 7.0.0. This setting was deprecated in 6.7.0. + +commit 023e3c207a748447a26be9700334ea4900c6f1fa +Author: Tim Brooks +Date: Sat Feb 9 21:19:57 2019 -0700 + + Concurrent file chunk fetching for CCR restore (#38656) + + Adds the ability to fetch chunks from different files in parallel, configurable using the new `ccr.indices.recovery.max_concurrent_file_chunks` setting, which defaults to 5 in this PR. + + The implementation uses the parallel file writer functionality that is also used by peer recoveries. + +commit c202900915c67f35fa5c0ade62a3acfe0528a118 +Author: Nhat Nguyen +Date: Sat Feb 9 19:51:58 2019 -0500 + + Retry on wait_for_metada_version timeout (#38521) + + Closes #37807 + Backport of #38521 + +commit 5b112b1d9db331fd9419814d0ff6891e6d56adbd +Author: Costin Leau +Date: Sun Feb 10 00:09:03 2019 +0200 + + SQL: remove beta marker from documentation (#38661) + + + + (cherry picked from commit fb6e7a30c9eed1e8b83496aaf1efe7e2288f9dd8) + +commit 56edc8e37f63f4ccb940d9abb8d8e51f48b7449f +Author: Alexander Reelsen +Date: Sat Feb 9 20:28:59 2019 +0100 + + Fix timezone fallback in ingest processor (#38407) (#38664) + + If no timezone was specified in the date processor, then the conversion + would lead to wrong time, as UTC was assumed by default, leading to + incorrectly parsed dates. + + This commit does not assume a default timezone and will thus not format + the dates in a wrong way. + +commit 5ab5a0a52961c6761cb5353f1a23653bf1800f28 +Author: Jason Tedor +Date: Sat Feb 9 09:22:43 2019 -0500 + + Avoid polluting download stats on builds (#38660) + + Recently we changed where we source released artifacts for usage in + backwards compatibility tests. We now source these from + artifacts.elastic.co. To avoid polluting the download stats from builds, + we want to add the X-Elastic-No-KPI header to requests from + artifacts.elastic.co. To do this, we hack the Ivy feature of custom HTTP + header credentials and specify our desired headers. + +commit 794ee4fb10a802f57a25e8523123f10851c42be5 +Author: Costin Leau +Date: Sat Feb 9 09:28:32 2019 +0200 + + SQL: Prevent grouping over grouping functions (#38649) + + Improve verifier to disallow grouping over grouping functions (e.g. + HISTOGRAM over HISTOGRAM). + + Close #38308 + + (cherry picked from commit 4e9b1cfd4df38c652bba36b4b4b538ce7c714b6e) + +commit 871036bd2161f5cdc22ee9727a21cd69323ccd07 +Author: Marios Trivyzas +Date: Sat Feb 9 09:17:31 2019 +0200 + + SQL: Relax StackOverflow circuit breaker for constants (#38572) + + Constant numbers (of any form: integers, decimals, negatives, + scientific) and strings shouldn't increase the depth counters + as they don't contribute to the increment of the stack depth. + + Fixes: #38571 + +commit 6abe99808a551fb70574c3ad0aa488ee14a905f7 +Author: Jason Tedor +Date: Fri Feb 8 19:47:33 2019 -0500 + + Drop support for the low-level REST client on JDK 7 (#38540) + + This commit bumps the minimum compiler version on the low-level REST + client to JDK 8. + +commit 46bd04959e1932c93496b1c9c887a3ab5058f761 +Author: Jake Landis +Date: Fri Feb 8 16:51:58 2019 -0600 + + fix dissect doc "ip" --> "clientip" (#38544) + + Forward port of #38512. + +commit af8a444caaef64516d1a1f08481db853759fdb70 +Author: Marios Trivyzas +Date: Fri Feb 8 22:55:51 2019 +0200 + + SQL: Replace joda with java time (#38437) + + Replace remaining usages of joda classes with java time. + + Fixes: #37703 + +commit 24a8ea06f57a9592ec45bda927cbc44b3d6be50c +Author: Benjamin Trent +Date: Fri Feb 8 12:56:04 2019 -0600 + + ML: update set_upgrade_mode, add logging (#38372) (#38538) + + * ML: update set_upgrade_mode, add logging + + * Attempt to fix datafeed isolation + + Also renamed a few methods/variables for clarity and added + some comments + +commit 9fd99f18a00397fc7a3df014b7905f40fb108de2 +Author: Jason Tedor +Date: Fri Feb 8 13:01:59 2019 -0500 + + Enable Dockerfile from artifacts.elastic.co (#38592) + + This commit enables the copyDockerfile task to render a Dockerfile that + sources the Elasticsearch binary from artifacts.elastic.co. This is + needed for reproducibility and transparency for the official Docker + images in the Docker library. + +commit 193464d8c8b0c7e6a643317639a528bb731c8b93 +Author: Tal Levy +Date: Fri Feb 8 09:40:53 2019 -0800 + + add geotile_grid ref to asciidoc (#38632) (#38639) + + Co-authored-by: Yuri Astrakhan + +commit d03b386f6ab27aa62f78f8db64f6b9f2dd24b08c +Author: Christoph Büscher +Date: Fri Feb 8 17:19:57 2019 +0100 + + Mute FollowerFailOverIT testFailOverOnFollower (#38634) + + Relates to #38633 + +commit 6359d988f02c583231879c83bec016a822d450aa +Author: Andrei Stefan +Date: Fri Feb 8 17:23:16 2019 +0200 + + Account for a possible rolled over file while reading the audit log file (#34909) + + (cherry picked from commit 75cb6b38ed67dc9d32c9291b0c174ffa94e473bc) + +commit e3c7b9391788f21334be0b85febeeb01b8ef861e +Author: Christoph Büscher +Date: Fri Feb 8 16:20:36 2019 +0100 + + Mute failure in InternalEngineTests (#38622) + +commit b284fede0b31cfcc3f4ce5fcd499be307b584e68 +Author: Martijn van Groningen +Date: Fri Feb 8 14:14:23 2019 +0100 + + Make qa/full-cluster-restart tests pass. By fixing a helper method and (#38604) + + muting a test. + + Relates to #38603 + +commit 779673c79265566ac670ad90b16059f37a752df3 +Author: Christoph Büscher +Date: Fri Feb 8 13:56:21 2019 +0100 + + Mute failing WatchStatusIntegrationTests (#38621) + + Relates to #38619 + +commit 5180b3654740ec78a488b7423c9f577c8163e5a2 +Author: Christoph Büscher +Date: Fri Feb 8 13:00:43 2019 +0100 + + Mute failing ApiKeyIntegTests (#38614) + +commit 02f57b1e299977a9eccf9bb793a7cc567d0fd2af +Author: David Roberts +Date: Fri Feb 8 11:35:37 2019 +0000 + + [DOCS] Add warning about bypassing ML PUT APIs (#38605) + + Now that ML configurations are stored in the .ml-config + index rather than in cluster state there is a possibility + that some users may try to add configurations directly to + the index. Allowing this creates a variety of problems + including possible data exflitration attacks (depending on + how security is set up), so this commit adds warnings + against allowing writes to the .ml-config index other than + via the ML APIs. + + Backport of #38509 + +commit fe8182ece2a426591984c5e09b6dd5bcb60c0133 +Author: Dimitris Athanasiou +Date: Fri Feb 8 11:32:28 2019 +0200 + + Mute RetentionLeastIT.testRetentionLeasesSyncOnRecovery on 7x (#38597) + +commit 1008f1c68e9be37d937dbe5214aea29b3f4edca2 +Author: Tim Vernum +Date: Fri Feb 8 18:06:15 2019 +1100 + + Only "include_type_name" if running on >= 7 (#38594) + + In cluster restart tests, we need to "include_type_name" if the + cluster includes a pre-7 version, but the test is running against + a 7+ version + +commit 84483b26cfd293fb90f4fa09c0cd55656ab6b684 +Author: Tim Vernum +Date: Fri Feb 8 18:00:03 2019 +1100 + + Fix version logic when bumping major version (#38593) + + When we are preparing to release a major version the rules around + "unreleased" versions and branches get a bit more complex. + + This change implements the following rules: + + - If the tip version on the previous major is a .0 (e.g. 6.7.0) then + the tip of the minor before that (e.g. 6.6.1) must be unreleased. + (This is because 6.7.0 would be "staged" in preparation for release, + but 6.6.1 would be open for bug fixes on the release 6.6.x line) + (in VersionCollection & VersionUtils) + + - The "major.x" branch (if it exists) will always point to the latest + minor in that series. Anything that is not the latest minor, must + therefore be on a the "major.minor" branch + For example, if v7.1.0 exists then the "7.x" branch must be 7.1.0, + and 7.0.0 must be on the "7.0" branch + (in VersionCollection) + +commit fdf6b3f23f0fdb0914107a69cbbbaadb262b08af +Author: Jason Tedor +Date: Thu Feb 7 16:32:27 2019 -0500 + + Add 7.1 version constant to 7.x branch (#38513) + + This commit adds the 7.1 version constant to the 7.x branch. + + Co-authored-by: Andy Bristol + Co-authored-by: Tim Brooks + Co-authored-by: Christoph Büscher + Co-authored-by: Luca Cavanna + Co-authored-by: markharwood + Co-authored-by: Ioannis Kakavas + Co-authored-by: Nhat Nguyen + Co-authored-by: David Roberts + Co-authored-by: Jason Tedor + Co-authored-by: Alpar Torok + Co-authored-by: David Turner + Co-authored-by: Martijn van Groningen + Co-authored-by: Tim Vernum + Co-authored-by: Albert Zaharovits + +commit 70956f6f34886cc48025d22ddce69b75d6b50c27 +Author: Lee Hinman +Date: Wed Feb 6 13:01:45 2019 -0700 + + bad formatted JSON object (#38515) (#38526) + + It just need to replace the wrong " , " to " : " + + Backport of #38515 + +commit f96bd2ad7177b0b9d3c5d00426f1240ccbce5d35 +Author: Marios Trivyzas +Date: Wed Feb 6 16:19:21 2019 +0200 + + SQL: Fix issue with IN not resolving to underlying keyword field (#38440) + + - Add resolution to the exact keyword field (if exists) for text fields. + - Add proper verification and error message if underlying keyword + doesn'texist. + - Move check for field attribute in the comparison list to the + `resolveType()` method of `IN`. + + Fixes: #38424 + +commit f8ed6c15c42a9cf990bc432d617e33eba3e218d4 +Author: Jason Tedor +Date: Wed Feb 6 08:03:19 2019 -0500 + + Enable BWC after backport recovering leases (#38485) + + This commit enables the BWC tests after backporting recovery of retention + leases during peer recovery. + +commit 4b42281a4e5baecc91e2fa7dac3076a48d9464c6 +Author: Jason Tedor +Date: Wed Feb 6 07:55:41 2019 -0500 + + Collapse retention lease integration tests (#38483) + + This commit collapses the retention lease integration tests into a + single suite. + +commit 510829f9f7198fa5d1f6a7a357a4802d861358bb +Author: Tanguy Leroux +Date: Wed Feb 6 13:22:54 2019 +0100 + + TransportVerifyShardBeforeCloseAction should force a flush (#38401) + + This commit changes the `TransportVerifyShardBeforeCloseAction` so that it + always forces the flush of the shard. It seems that #37961 is not sufficient to + ensure that the translog and the Lucene commit share the exact same max + seq no and global checkpoint information in case of one or more noop + operations have been made. + + The `BulkWithUpdatesIT.testThatMissingIndexDoesNotAbortFullBulkRequest` + and `FrozenIndexTests.testFreezeEmptyIndexWithTranslogOps` test this trivial + situation and they both fail 1 on 10 executions. + + Relates to #33888 + +commit 5a3c4524805f88f0875989d656f87c01595a8de7 +Author: David Turner +Date: Wed Feb 6 11:34:38 2019 +0000 + + Align docs etc with new discovery setting names (#38492) + + In #38333 and #38350 we moved away from the `discovery.zen` settings namespace + since these settings have an effect even though Zen Discovery itself is being + phased out. This change aligns the documentation and the names of related + classes and methods with the newly-introduced naming conventions. + +commit 1a02445ae17b543f044259803f59fbdbb7947e45 +Author: Costin Leau +Date: Wed Feb 6 12:08:32 2019 +0200 + + SQL: Allow look-ahead resolution of aliases for WHERE clause (#38450) + + Aliases defined in SELECT (Project or Aggregate) are now resolved in the + following WHERE clause. The Analyzer has been enhanced to identify this + rule and replace the field accordingly. + + Close #29983 + +commit 6ff4a8cfd51a5683b80cd163e11702b843e68254 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Feb 6 20:58:22 2019 +1100 + + Add API key settings documentation (#38490) + + This commit adds missing + API key service settings documentation. + +commit a7046e001ce6b5397deee35c0a0e348291ffe034 +Author: Luca Cavanna +Date: Wed Feb 6 08:43:47 2019 +0100 + + Remove support for maxRetryTimeout from low-level REST client (#38085) + + We have had various reports of problems caused by the maxRetryTimeout + setting in the low-level REST client. Such setting was initially added + in the attempts to not have requests go through retries if the request + already took longer than the provided timeout. + + The implementation was problematic though as such timeout would also + expire in the first request attempt (see #31834), would leave the + request executing after expiration causing memory leaks (see #33342), + and would not take into account the http client internal queuing (see #25951). + + Given all these issues, it seems that this custom timeout mechanism + gives little benefits while causing a lot of harm. We should rather rely + on connect and socket timeout exposed by the underlying http client + and accept that a request can overall take longer than the configured + timeout, which is the case even with a single retry anyways. + + This commit removes the `maxRetryTimeout` setting and all of its usages. + +commit 9492dcea42d0aa61658c11f7432107972cd30b7d +Author: Martijn van Groningen +Date: Wed Feb 6 08:21:30 2019 +0100 + + Update IndexTemplateMetaData to allow unknown fields (#38448) + + Updated IndexTemplateMetaData to use ObjectParser. + + The IndexTemplateMetaData class used old parsing logic and was not + resiliant to new fields. This commit updates it to use the + ConstructingObjectParser and allow unknown fields. + + Relates #36938 + + Co-authored-by: Michael Basnight + Co-authored-by: Martijn van Groningen + +commit e1d464b22c3352ec535de536c9cabc8ad1f367f5 +Author: Ioannis Kakavas +Date: Wed Feb 6 08:52:54 2019 +0200 + + Mute testRetentionLeasesSyncOnRecovery (#38488) + + Relates: #38487 + +commit 5261673349b4d16c0597071c0d3ef1c3e64e157b +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Feb 6 16:03:49 2019 +1100 + + Change the min supported version to 6.7.0 for API keys (#38481) + + This commit changes the minimum supported version to 6.7.0 + for API keys, the change for the API keys has been backported + to 6.7.0 version #38399 + +commit e73c9c90ee8a77c8b97e6bbcfc540bb7a00b2dae +Author: Jay Modi +Date: Tue Feb 5 18:16:26 2019 -0700 + + Add an authentication cache for API keys (#38469) + + This commit adds an authentication cache for API keys that caches the + hash of an API key with a faster hash. This will enable better + performance when API keys are used for bulk or heavy searching. + +commit 517aa959845f0f1b52406f6d13c2f84af8fff49e +Author: Tim Vernum +Date: Wed Feb 6 12:05:44 2019 +1100 + + Fix exit code in certutil packaging test (#38393) + + The exit code is different on windows, and we don't really care about + which code it is, we just need to check that it's not 0 (success) + +commit 57600c5acb876129158b97885820833da054cdc6 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Feb 6 11:21:54 2019 +1100 + + Enable logs for intermittent test failure (#38426) + + I have not been able to reproduce the failing + test scenario locally for #38408 and there are other similar + tests which are running fine in the same test class. + I am re-enabling the test with additional logs so + that we can debug further on what's happening. + I will keep the issue open for now and look out for the builds + to see if there are any related failures. + +commit 1aa32cac8fc2f50458d2c07b557befdac88be2de +Author: Jason Tedor +Date: Tue Feb 5 19:16:25 2019 -0500 + + Disable BWC to backport recovering retention leases (#38477) + + This commit disable the BWC tests in preparation for backporting + recovery of retention leases during peer recovery. + +commit 8972ebabddc3c69fcaf40887f6066961c391b70f +Author: Martijn van Groningen +Date: Wed Feb 6 00:04:43 2019 +0100 + + Enable bwc tests now that #38443 is backported. (#38462) + +commit 34f2cc78f661170ee3b3b2e703e6e06632189ef7 +Author: Armin Braun +Date: Tue Feb 5 23:56:59 2019 +0100 + + Fix Master Failover and DataNode Leave Blocking Snapshot (#38460) + + * Closes #38447 + +commit 79a45b47da618bd4529ef67051d6f74e2da721ce +Author: Jason Tedor +Date: Tue Feb 5 17:43:41 2019 -0500 + + Recover retention leases during peer recovery (#38435) + + This commit integrates retention leases with recovery. With this change, + we copy the current retention leases on primary to the replica during + phase two of recovery. At this point, the replica will have been added + to the replication group and so is already receiving retention lease + sync requests from the primary. This means that if any retention lease + syncs are triggered on the primary after we sample the retention leases + here during phase two, that sync request will also arrive on the replica + ensuring that the replica is from this point on up to date with the + retention leases on the primary. We have to copy these during phase two + since we will be applying indexing operations, potentially triggering + merges, and therefore must ensure the correct retention leases are in + place beforehand. + +commit fb0ec26fd4d5f0e53bc79ae06ba3f68c6454bd0d +Author: Tim Brooks +Date: Tue Feb 5 16:22:11 2019 -0600 + + Set update mappings mater node timeout to 30 min (#38439) + + This is related to #35975. We do not want a slow master to fail a + recovery from remote process due to a slow put mappings call. This + commit increases the master node timeout on this call to 30 mins. + +commit f939c3c5ef86278f735f9f5e812f1282606eba3f +Author: Zachary Tong +Date: Tue Feb 5 17:06:28 2019 -0500 + + Assert job is not null in FullClusterRestartIT (#38218) + + `waitForRollUpJob` is an assertBusy that waits for the rollup job + to appear in the tasks list, and waits for it to be a certain state. + + However, there was a null check around the state assertion, which meant + if the job _was_ null, the assertion would be skipped, and the + assertBusy would pass withouot an exception. This could then lead to + downstream assertions to fail because the job was not actually ready, + or in the wrong state. + + This changes the test to assert the job is not null, so the assertBusy + operates as intended. + +commit 2b6b85815b52009240f29116fd7684d764ba13ae +Author: Lee Hinman +Date: Tue Feb 5 14:42:31 2019 -0700 + + Update ilm-api.asciidoc, point to REMOVE policy (#38235) (#38463) + +commit 2c30501c74c195e6c289ad5dc9fd568557808a59 +Author: Marios Trivyzas +Date: Tue Feb 5 23:12:52 2019 +0200 + + SQL: Fix esType for DATETIME/DATE and INTERVALS (#38179) + + Since introduction of data types that don't have a corresponding type + in ES the `esType` is error-prone when used for `unmappedType()` calls. + Moreover since the renaming of `DATE` to `DATETIME` and the introduction + of an actual date-only `DATE` the `esType` would return `datetime` which + is not a valid type for ES mapping. + + Fixes: #38051 + +commit 1f4f6f35c8db0bb9a8fb94e0277b2ac69d2d3729 +Author: Ioannis Kakavas +Date: Tue Feb 5 23:11:21 2019 +0200 + + Handle deprecation header-AbstractUpgradeTestCase (#38396) + +commit afcdbd2bc03a9df3b5a45b6f7dfcc2b5e52df404 +Author: Przemyslaw Gomulka +Date: Tue Feb 5 22:09:32 2019 +0100 + + XPack: core/ccr/Security-cli migration to java-time (#38415) + + part of the migrating joda time work. + refactoring x-pack plugins usages of joda to java-time + refers #27330 + +commit 68cb7b98796847dacf479990a8de3f88544401ba +Author: Martijn van Groningen +Date: Tue Feb 5 22:04:18 2019 +0100 + + Disable bwc tests for #38443 (#38456) + +commit 20c66c5a057c06c56c446636dc684434ca6c08c1 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Feb 5 21:48:24 2019 +0100 + + Bubble-up exceptions from scheduler (#38317) + + Instead of logging warnings we now rethrow exceptions thrown inside + scheduled/submitted tasks. This will still log them as warnings in + production but has the added benefit that if they are thrown during + unit/integration test runs, the test will be flagged as an error. + + This is a continuation of #38014 + + Fixed NPE that caused CCR tests (IndexFollowingIT and likely others) + to fail. + + schedule could bubble rejected exception to uncaught exception + handler when not using SAME executor if thread pool is terminated. + Now ignore rejected exception silently if executor is shutdown. + +commit a6ce671751fd219ecefd0d0c963a24489431c910 +Author: Lee Hinman +Date: Tue Feb 5 13:42:43 2019 -0700 + + Re-enable TasksClientDocumentationIT.testCancelTasks (#38234) + + This test has been disabled since November 2018, but I was not able to reproduce + the failure. Re-enabling this so we can see the full log and get more context if + it fails again. + + Relates to #35514 + +commit 7ca5495d86093cd4673f7a96f6752406ac82cfda +Author: Jay Modi +Date: Tue Feb 5 13:39:29 2019 -0700 + + Allow custom authorization with an authorization engine (#38358) + + For some users, the built in authorization mechanism does not fit their + needs and no feature that we offer would allow them to control the + authorization process to meet their needs. In order to support this, + a concept of an AuthorizationEngine is being introduced, which can be + provided using the security extension mechanism. + + An AuthorizationEngine is responsible for making the authorization + decisions about a request. The engine is responsible for knowing how to + authorize and can be backed by whatever mechanism a user wants. The + default mechanism is one backed by roles to provide the authorization + decisions. The AuthorizationEngine will be called by the + AuthorizationService, which handles more of the internal workings that + apply in general to authorization within Elasticsearch. + + In order to support external authorization services that would back an + authorization engine, the entire authorization process has become + asynchronous, which also includes all calls to the AuthorizationEngine. + + The use of roles also leaked out of the AuthorizationService in our + existing code that is not specifically related to roles so this also + needed to be addressed. RequestInterceptor instances sometimes used a + role to ensure a user was not attempting to escalate their privileges. + Addressing this leakage of roles meant that the RequestInterceptor + execution needed to move within the AuthorizationService and that + AuthorizationEngines needed to support detection of whether a user has + more privileges on a name than another. The second area where roles + leaked to the user is in the handling of a few privilege APIs that + could be used to retrieve the user's privileges or ask if a user has + privileges to perform an action. To remove the leakage of roles from + these actions, the AuthorizationService and AuthorizationEngine gained + methods that enabled an AuthorizationEngine to return the response for + these APIs. + + Ultimately this feature is the work included in: + #37785 + #37495 + #37328 + #36245 + #38137 + #38219 + + Closes #32435 + +commit 8ad9a07b87432f3f8586f974e9a74f9ab25bd209 +Author: Boaz Leskes +Date: Tue Feb 5 21:27:28 2019 +0100 + + CRUDDocumentationIT fix documentation references + +commit 033ba725af26b940226aaa9b1ce88349bc433af6 +Author: Boaz Leskes +Date: Tue Feb 5 20:53:35 2019 +0100 + + Remove support for internal versioning for concurrency control (#38254) + + Elasticsearch has long [supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-versioning) compare and set (a.k.a optimistic concurrency control) operations using internal document versioning. Sadly that approach is flawed and can sometime do the wrong thing. Here's the relevant excerpt from the resiliency status page: + + > When a primary has been partitioned away from the cluster there is a short period of time until it detects this. During that time it will continue indexing writes locally, thereby updating document versions. When it tries to replicate the operation, however, it will discover that it is partitioned away. It won’t acknowledge the write and will wait until the partition is resolved to negotiate with the master on how to proceed. The master will decide to either fail any replicas which failed to index the operations on the primary or tell the primary that it has to step down because a new primary has been chosen in the meantime. Since the old primary has already written documents, clients may already have read from the old primary before it shuts itself down. The version numbers of these reads may not be unique if the new primary has already accepted writes for the same document + + We recently [introduced](https://www.elastic.co/guide/en/elasticsearch/reference/6.x/optimistic-concurrency-control.html) a new sequence number based approach that doesn't suffer from this dirty reads problem. + + This commit removes support for internal versioning as a concurrency control mechanism in favor of the sequence number approach. + + Relates to #1078 + +commit b03d138122d61c9ee074b747ce040975e9e300b3 +Author: Jason Tedor +Date: Tue Feb 5 14:42:17 2019 -0500 + + Lift retention lease expiration to index shard (#38380) + + This commit lifts the control of when retention leases are expired to + index shard. In this case, we move expiration to an explicit action + rather than a side-effect of calling + ReplicationTracker#getRetentionLeases. This explicit action is invoked + on a timer. If any retention leases expire, then we hard sync the + retention leases to the replicas. Otherwise, we proceed with a + background sync. + +commit 4a15e2b29e9fdca39bb5e224415f653eeae5fdf4 +Author: Tim Brooks +Date: Tue Feb 5 13:34:00 2019 -0600 + + Make Ccr recovery file chunk size configurable (#38370) + + This commit adds a byte setting `ccr.indices.recovery.chunk_size`. This + setting configs the size of file chunk requested while recovering from + remote. + +commit c2a8fe1f91e2cec7c930463f65987df4e4c094bf +Author: Tim Brooks +Date: Tue Feb 5 13:32:41 2019 -0600 + + Prevent CCR recovery from missing documents (#38237) + + Currently the snapshot/restore process manually sets the global + checkpoint to the max sequence number from the restored segements. This + does not work for Ccr as this will lead to documents that would be + recovered in the normal followering operation from being recovered. + + This commit fixes this issue by setting the initial global checkpoint to + the existing local checkpoint. + +commit aef5775561deaf64d3f877ec4b8a4cc36f40f62e +Author: Tal Levy +Date: Tue Feb 5 11:20:40 2019 -0800 + + re-enables awaitsfixed datemath tests (#38376) + + Previously, date formats of `YYYY.MM.dd` would hit an issue + where the year would jump towards the end of the calendar year. + This was an issue that had since been resolved in tests by using + `yyyy` to be the more accurate representation of the year. + + Closes #37037. + +commit e4fa32470bfe19776715400f5d983f3f877afc8f +Author: Mayya Sharipova +Date: Tue Feb 5 14:15:43 2019 -0500 + + Types removal fix FullClusterRestartIT warnings (#38445) + + Backport PR #38389 for 6.7 produces warnings for rollover test. + This fixes FullClusterRestartIT warning expectations + for rollover request + + Relates to #38389 + +commit 3ce7d2c9b621e796ace43008134413cea7648a06 +Author: Julie Tibshirani +Date: Tue Feb 5 10:52:32 2019 -0800 + + Make sure to reject mappings with type _doc when include_type_name is false. (#38270) + + `CreateIndexRequest#source(Map, ... )`, which is used when + deserializing index creation requests, accidentally accepts mappings that are + nested twice under the type key (as described in the bug report #38266). + + This in turn causes us to be too lenient in parsing typeless mappings. In + particular, we accept the following index creation request, even though it + should not contain the type key `_doc`: + + ``` + PUT index?include_type_name=false + { + "mappings": { + "_doc": { + "properties": { ... } + } + } + } + ``` + + There is a similar issue for both 'put templates' and 'put mappings' requests + as well. + + This PR makes the minimal changes to detect and reject these typed mappings in + requests. It does not address #38266 generally, or attempt a larger refactor + around types in these server-side requests, as I think this should be done at a + later time. + +commit 8ebff0512b7a42f67f196f2bfb62ab00a66ea765 +Author: austintp +Date: Tue Feb 5 12:37:02 2019 -0600 + + Updates the grok patterns to be consistent with logstash (#27181) + +commit ca47f680912ddf3d67465ad0722b180a8e70d20d +Author: Christoph Büscher +Date: Tue Feb 5 19:07:53 2019 +0100 + + Ignore type-removal warnings in XPackRestTestHelper (#38431) + + The backport of #38022 introduced types-deprecation warning for get/put template requests + that cause problems on tests master in mixed cluster scenarios. While these warnings are + caught and ignored in regular Rest tests, the get template requests in XPackRestTestHelper + were missed. + + Closes #38412 + +commit 54e684bedde2d863247ffe68a536e525fb706ee7 +Author: Zachary Tong +Date: Tue Feb 5 12:59:05 2019 -0500 + + testHlrcFromXContent() should respect assertToXContentEquivalence() (#38232) + + Tests can override assertToXContentEquivalence() in case their xcontent + cannot be directly compared (e.g. due to insertion order in maps + affecting the xcontent ordering). But the `testHlrcFromXContent` test + hardcoded the equivalence test to `true` instead of consulting + `assertToXContentEquivalence()` + + Fixes #36034 + +commit ae47c025e225eec2a75d63c65ff18378124c4722 +Author: Tal Levy +Date: Tue Feb 5 09:44:47 2019 -0800 + + add basic REST test for geohash_grid (#37996) + +commit f2dd5dd6eb6cbaffe86967875faaf8a0148b9e4e +Author: David Turner +Date: Tue Feb 5 17:42:24 2019 +0000 + + Remove DiscoveryPlugin#getDiscoveryTypes (#38414) + + With this change we no longer support pluggable discovery implementations. No + known implementations of `DiscoveryPlugin` actually override this method, so in + practice this should have no effect on the wider world. However, we were using + this rather extensively in tests to provide the `test-zen` discovery type. We + no longer need a separate discovery type for tests as we no longer need to + customise its behaviour. + + Relates #38410 + +commit 963b474f2f2ab2edfbbb22c1eedebe7c841cd5ae +Author: Przemyslaw Gomulka +Date: Tue Feb 5 18:27:24 2019 +0100 + + Fix the clock resolution to millis in GetWatchResponseTests (#38405) + + the clock resolution changed from jdk8->jdk10, hence the test is passing + in jdk8 but failing in jdk10. The Watcher's objects are serialised and + deserialised with milliseconds precision, making test to fail in jdk 10 + and higher + + closes #38400 + +commit b7ab521eb1e78e7bbc84738c9c6c5cda85cbbfb0 +Author: David Turner +Date: Tue Feb 5 17:11:20 2019 +0000 + + Throw AssertionError when no master (#38432) + + Today we throw a fatal `RuntimeException` if an exception occurs in + `getMasterName()`, and this includes the case where there is currently no + master. However, sometimes we call this method inside an `assertBusy()` in + order to allow for a cluster that is in the process of stabilising and electing + a master. The trouble is that `assertBusy()` only retries on an + `AssertionError` and not on a general `RuntimeException`, so the lack of a + master is immediately fatal. + + This commit fixes the issue by asserting there is a master, triggering a retry + if there is not. + + Fixes #38331 + +commit 12657fda4498d11e3a0102d52af469d0b9907c20 +Author: Boaz Leskes +Date: Tue Feb 5 18:05:56 2019 +0100 + + `if_seq_no` and `if_primary_term` parameters aren't wired correctly in REST Client's CRUD API (#38411) + +commit df4eb0485ddd52631830e86985373a3799d3df12 +Author: Przemyslaw Gomulka +Date: Tue Feb 5 17:48:47 2019 +0100 + + Enable CronEvalToolTest.testEnsureDateIsShownInRootLocale (#38394) + + The test is now expected to be always passing no matter what the random + locale is. This is fixed with using jdk ZoneId.systemDefault() in both + the test and CronEvalTool + + closes #35687 + +commit 440d1eda8a51f94aa6effaff7944d958b293d1ca +Author: Julie Tibshirani +Date: Tue Feb 5 08:42:37 2019 -0800 + + Fix failures in BulkProcessorIT#testGlobalParametersAndBulkProcessor. (#38129) + + This PR fixes a couple test issues: + * It narrows an assertWarnings call that was too broad, and wasn't always + applicable with certain random sequences. + * Previously, we could send a typeless bulk request containing '_type: 'null'. + Now we omit the _type key altogether for typeless requests. + +commit c9701be1e836d154a0758e609d2d0735b03a5231 +Author: Marios Trivyzas +Date: Tue Feb 5 18:15:26 2019 +0200 + + SQL: Implement CURRENT_DATE (#38175) + + Since DATE data type is now available, this implements the + `CURRENT_DATE/CURRENT_DATE()/TODAY()` similar to `CURRENT_TIMESTAMP`. + + Closes: #38160 + +commit 887fa2c97abba907d56271a21a2c0bbd141f58dd +Author: Armin Braun +Date: Tue Feb 5 17:10:12 2019 +0100 + + Mute testReadRequestsReturnLatestMappingVersion (#38438) + + * Relates #37807 + +commit 92bc68170548dfa3587cbf5058bb10bc58503e6c +Author: David Roberts +Date: Tue Feb 5 16:10:00 2019 +0000 + + [ML] Report index unavailable instead of waiting for lazy node (#38423) + + If a job cannot be assigned to a node because an index it + requires is unavailable and there are lazy ML nodes then + index unavailable should be reported as the assignment + explanation rather than waiting for a lazy ML node. + +commit 8742db3afe70d8dea9db8a1f9a281ea65f6e77c8 +Author: Michael Basnight +Date: Tue Feb 5 10:08:08 2019 -0600 + + Update Rollup Caps to allow unknown fields (#38339) + + This commit ensures that the parts of rollup caps that can allow unknown + fields will allow them. It also modifies the test such that we can use + the features we need for disallowing fields in spots where they would + not be allowed. + + Relates #36938 + +commit 0beb3c93d1ca6dd891269d902bde0e32b9983bbc +Author: Martijn van Groningen +Date: Tue Feb 5 17:05:19 2019 +0100 + + Clean up duplicate follow config parameter code (#37688) + + Introduced FollowParameters class that put follow, resume follow, + put auto follow pattern requests and follow info response classes reuse. + + The FollowParameters class had the fields, getters etc. for the common parameters + that all these APIs have. Also binary and xcontent serialization / + parsing is handled by this class. + + The follow, resume follow, put auto follow pattern request classes originally + used optional non primitive fields, so FollowParameters has that too and the follow info api can handle that now too. + + Also the followerIndex field can in production only be specified via + the url path. If it is also specified via the request body then + it must have the same value as is specified in the url path. This + option only existed to xcontent testing. However the AbstractSerializingTestCase + base class now also supports createXContextTestInstance() to provide + a different test instance when testing xcontent, so allowing followerIndex + to be specified via the request body is no longer needed. + + By moving the followerIndex field from Body to ResumeFollowAction.Request + class and not allowing the followerIndex field to be specified via + the request body the Body class is redundant and can be removed. The + ResumeFollowAction.Request class can then directly use the + FollowParameters class. + + For consistency I also removed the ability to specified followerIndex + in the put follow api and the name in put auto follow pattern api via + the request body. + +commit 2f6afd290e4a884ea4b653fe4522917eceeb8d39 +Author: Armin Braun +Date: Tue Feb 5 16:44:18 2019 +0100 + + Fix Concurrent Snapshot Ending And Stabilize Snapshot Finalization (#38368) + + * The problem in #38226 is that in some corner cases multiple calls to `endSnapshot` were made concurrently, leading to non-deterministic behavior (`beginSnapshot` was triggering a repository finalization while one that was triggered by a `deleteSnapshot` was already in progress) + * Fixed by: + * Making all `endSnapshot` calls originate from the cluster state being in a "completed" state (apart from on short-circuit on initializing an empty snapshot). This forced putting the failure string into `SnapshotsInProgress.Entry`. + * Adding deduplication logic to `endSnapshot` + * Also: + * Streamlined the init behavior to work the same way (keep state on the `SnapshotsService` to decide which snapshot entries are stale) + * closes #38226 + +commit d862453d6822525fc7b14ec60a5496767dd7fa8e +Author: Lee Hinman +Date: Tue Feb 5 07:52:17 2019 -0700 + + Support unknown fields in ingest pipeline map configuration (#38352) + + We already support unknown objects in the list of pipelines, this changes the + `PipelineConfiguration` to support fields other than just `id` and `config`. + + Relates to #36938 + +commit 638ba4a59a6663ebfd4fd56a7544283e7a51ca7f +Author: Jason Tedor +Date: Tue Feb 5 06:08:03 2019 -0500 + + Mute failing API key integration test (#38409) + + This commit mutes the test + testGetAndInvalidateApiKeysWithExpiredAndInvalidatedApiKey as it failed + during a PR build. + +commit cea81b199dcb92bf80fb42a87fb4817a4f63d890 +Author: Andrei Stefan +Date: Tue Feb 5 12:00:49 2019 +0200 + + Change the milliseconds precision to 3 digits for intervals. (#38297) + +commit 8e2eb39cef53f762e78b3d4b90755996da97120b +Author: Albert Zaharovits +Date: Tue Feb 5 10:49:08 2019 +0200 + + SecuritySettingsSource license.self_generated: trial (#38233) + + Authn is enabled only if `license_type` is non `basic`, but `basic` is + what the `LicenseService` generates implicitly. This commit explicitly sets + license type to `trial`, which allows for authn, in the `SecuritySettingsSource` + which is the settings configuration parameter for `InternalTestCluster`s. + + The real problem, that had created tests failures like #31028 and #32685, is + that the check `licenseState.isAuthAllowed()` can change sporadically. If it were + to return `true` or `false` during the whole test there would be no problem. + The problem manifests when it turns from `true` to `false` right before `Realms.asList()`. + There are other license checks before this one (request filter, token service, etc) + that would not cause a problem if they would suddenly see the check as `false`. + But switching to `false` before `Realms.asList()` makes it appear that no installed + realms could have handled the authn token which is an authentication error, as can + be seen in the failing tests. + + Closes #31028 #32685 + +commit 3b2a0d79597c8769d04e8f8cf7e44e59397bf6b3 +Author: David Turner +Date: Tue Feb 5 08:47:56 2019 +0000 + + Rename no-master-block setting (#38350) + + Replaces `discovery.zen.no_master_block` with `cluster.no_master_block`. Any + value set for the old setting is now ignored. + +commit 2d114a02fffb75c3088561c9ec6ab6fb67e8d980 +Author: David Turner +Date: Tue Feb 5 08:46:52 2019 +0000 + + Rename static Zen1 settings (#38333) + + Renames the following settings to remove the mention of `zen` in their names: + + - `discovery.zen.hosts_provider` -> `discovery.seed_providers` + - `discovery.zen.ping.unicast.concurrent_connects` -> `discovery.seed_resolver.max_concurrent_resolvers` + - `discovery.zen.ping.unicast.hosts.resolve_timeout` -> `discovery.seed_resolver.timeout` + - `discovery.zen.ping.unicast.hosts` -> `discovery.seed_addresses` + +commit 89feaa0e23ee4638668bbfcca11465d959c5bea9 +Author: Przemyslaw Gomulka +Date: Tue Feb 5 09:09:15 2019 +0100 + + Migration doc for audit json log file (#38165) + + The migration documentation for an audit logging changes. Removal of plaintext logs and rename of json log file + + relates #32850 + +commit 64ff75f04e88e2441a275e3c207a251fe66d94a7 +Author: Brandon Kobel +Date: Mon Feb 4 21:45:28 2019 -0800 + + Add apm_user reserved role (#38206) + + * Adding apm_user + + * Fixing SecurityDocumentationIT testGetRoles test + + * Adding access to .ml-anomalies-* + + * Fixing APM test, we don't have access to the ML state index + +commit fe36861ada3e75259ff601a9f6b0e1cd5fbb8afe +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Feb 5 14:21:57 2019 +1100 + + Add support for API keys to access Elasticsearch (#38291) + + X-Pack security supports built-in authentication service + `token-service` that allows access tokens to be used to + access Elasticsearch without using Basic authentication. + The tokens are generated by `token-service` based on + OAuth2 spec. The access token is a short-lived token + (defaults to 20m) and refresh token with a lifetime of 24 hours, + making them unsuitable for long-lived or recurring tasks where + the system might go offline thereby failing refresh of tokens. + + This commit introduces a built-in authentication service + `api-key-service` that adds support for long-lived tokens aka API + keys to access Elasticsearch. The `api-key-service` is consulted + after `token-service` in the authentication chain. By default, + if TLS is enabled then `api-key-service` is also enabled. + The service can be disabled using the configuration setting. + + The API keys:- + - by default do not have an expiration but expiration can be + configured where the API keys need to be expired after a + certain amount of time. + - when generated will keep authentication information of the user that + generated them. + - can be defined with a role describing the privileges for accessing + Elasticsearch and will be limited by the role of the user that + generated them + - can be invalidated via invalidation API + - information can be retrieved via a get API + - that have been expired or invalidated will be retained for 1 week + before being deleted. The expired API keys remover task handles this. + + Following are the API key management APIs:- + 1. Create API Key - `PUT/POST /_security/api_key` + 2. Get API key(s) - `GET /_security/api_key` + 3. Invalidate API Key(s) `DELETE /_security/api_key` + + The API keys can be used to access Elasticsearch using `Authorization` + header, where the auth scheme is `ApiKey` and the credentials, is the + base64 encoding of API key Id and API key separated by a colon. + Example:- + ``` + curl -H "Authorization: ApiKey YXBpLWtleS1pZDphcGkta2V5" http://localhost:9200/_cluster/health + ``` + + Closes #34383 + +commit d25530358452b4abbf223017003692ea92e76f8e +Author: Christoph Büscher +Date: Tue Feb 5 03:41:05 2019 +0100 + + Add typless client side GetIndexRequest calls and response class (#37778) + + The HLRC client currently uses `org.elasticsearch.action.admin.indices.get.GetIndexRequest` + and `org.elasticsearch.action.admin.indices.get.GetIndexResponse` in its get index calls. Both request and + response are designed for the typed APIs, including some return types e.g. for `getMappings()` which in + the maps it returns still use a level including the type name. + In order to change this without breaking existing users of the HLRC API, this PR introduces two new request + and response objects in the `org.elasticsearch.client.indices` client package. These are used by the + IndicesClient#get and IndicesClient#exists calls now by default and support the type-less API. The old request + and response objects are still kept for use in similarly named, but deprecated methods. + + The newly introduced client side classes are simplified versions of the server side request/response classes since + they don't need to support wire serialization, and only the response needs fromXContent parsing (but no + xContent-serialization, since this is the responsibility of the server-side class). + Also changing the return type of `GetIndexResponse#getMapping` to + `Map getMappings()`, while it previously was returning another map + keyed by the type-name. Similar getters return simple Maps instead of the ImmutableOpenMaps that the + server side response objects return. + +commit 9d3f0578943063f5f6858cced3fb5289505d8b9c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Feb 5 12:02:36 2019 +1100 + + Limit token expiry to 1 hour maximum (#38244) + + We mention in our documentation for the token + expiration configuration maximum value is 1 hour + but do not enforce it. This commit adds max limit + to the TOKEN_EXPIRATION setting. + +commit 48f09471f8c091b96c6a36ed7d476ebad69e331e +Author: Tal Levy +Date: Mon Feb 4 17:00:40 2019 -0800 + + add docs saying mixed-cluster ILM is not supported (#37954) + + Closes #37085. + +commit b5b319ec9a21e85ff0978fce74c36db14b054e7a +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Feb 5 11:01:13 2019 +1100 + + Skip unsupported languages for tests (#38328) + + Skip the languages in tests for which SimpleKdcServer + does not handle generalized time correctly. + + Closes#38320 + +commit 292e0f6fb76a1e280580cb4d5798a7cc876eb8ae +Author: Gordon Brown +Date: Mon Feb 4 16:11:44 2019 -0700 + + Deprecate `_type` in simulate pipeline requests (#37949) + + As mapping types are being removed throughout Elasticsearch, the use of + `_type` in pipeline simulation requests is deprecated. Additionally, the + default `_type` used if one is not supplied has been changed to `_doc` for + consistency with the rest of Elasticsearch. + +commit b866417650e450e6b9e478e6711b22fec59bfffd +Author: Gordon Brown +Date: Mon Feb 4 16:06:19 2019 -0700 + + Mute testCannotShrinkLeaderIndex (#38374) + + This test should not pass until CCR finishes integrating shard history + retention leases. It currently sometimes passes (which is a bug in the + test), but cannot pass reliably until the linked issue is resolved. + +commit cecfa5bd6def76e2799f2d7a1b96a80f0220dac2 +Author: Nhat Nguyen +Date: Mon Feb 4 17:53:41 2019 -0500 + + Tighten mapping syncing in ccr remote restore (#38071) + + There are two issues regarding the way that we sync mapping from leader + to follower when a ccr restore is completed: + + 1. The returned mapping from a cluster service might not be up to date + as the mapping of the restored index commit. + + 2. We should not compare the mapping version of the follower and the + leader. They are not related to one another. + + Moreover, I think we should only ensure that once the restore is done, + the mapping on the follower should be at least the mapping of the copied + index commit. We don't have to sync the mapping which is updated after + we have opened a session. + + Relates #36879 + Closes #37887 + +commit 5a33816c86e4fdf0459be428ebe2328e3e4ede20 +Author: Tim Brooks +Date: Mon Feb 4 16:37:42 2019 -0600 + + Add test for `PutFollowAction` on a closed index (#38236) + + This is related to #35975. Currently when an index falls behind a leader + it encounters a fatal exception. This commit adds a test for that + scenario. Additionally, it tests that the user can stop following, close + the follower index, and put follow again. After the indexing is + re-bootstrapped, it will recover the documents it lost in normal + following operations. + +commit c3cdf84c04e6137da256561d773df0a9a066c8ec +Author: Jay Modi +Date: Mon Feb 4 14:34:37 2019 -0700 + + Fix SSLContext pinning to TLSV1.2 in reload tests (#38341) + + This commit fixes the pinning of SSLContexts to TLSv1.2 in the + SSLConfigurationReloaderTests. The pinning was added for the initial + creation of clients and webservers but the updated contexts would + default to TLSv1.3, which is known to cause hangs with the + MockWebServer that we use. + + Relates #38103 + Closes #38247 + +commit 0ced7753891489e3b26cf3552b1e05bbb2149f1c +Author: Christoph Büscher +Date: Mon Feb 4 22:30:34 2019 +0100 + + Mute RareClusterStateIT.testDelayedMappingPropagationOnReplica (#38357) + +commit 641704464d7b59ef60c215efda2222bec0bb345d +Author: Mayya Sharipova +Date: Mon Feb 4 16:07:45 2019 -0500 + + Deprecate types in rollover index API (#38039) + + Relates to #35190 + +commit 578fd14257b5881bf28149e90382a4ce8b5d2306 +Author: markharwood +Date: Mon Feb 4 20:09:07 2019 +0000 + + Types removal - fix FullClusterRestartIT warning expectations (#38310) + + Relax test warning message checking to pre-empt PR 38022 landing in 6.7 with new warning messages. + The relaxed test now just assumes any warning message starting with “[types removal]” is tolerated rather than the precise phrasing used in the 6.7 branch. + +commit 24db0534650ebcf3ca3e3002109511de19b2a4ca +Author: Michael Basnight +Date: Mon Feb 4 14:05:00 2019 -0600 + + Fix ILM explain response to allow unknown fields (#38054) + + IndexLifecycleExplainResponse did not allow unknown fields. This commit + fixes the test and ConstructingObjectParser such that it allows unknown + fields. + +commit fb1e350c815c6f5a1a75a80ddf34eb9dc7e1bb80 +Author: Nhat Nguyen +Date: Mon Feb 4 15:04:46 2019 -0500 + + Mute testFollowIndexAndCloseNode (#38360) + + Tracked at #33337 + +commit 5d949dddfb76e35a3c1cdcf85f86d9b7bd6cd594 +Author: Nik Everett +Date: Mon Feb 4 14:57:38 2019 -0500 + + Docs: Drop inline callout from scroll example (#38340) + + Coalesces two calls into one in a scroll example so all callouts are at + the end of the line. This is the only sort of callouts that are + supported by asciidoctor and we'd like to start building our docs with + asciidoctor. + + At present we don't have any mechanism to stop folks adding more inline + callouts but we ought to be able to have one in a few weeks. For now, + though, removing these inline callouts is a step in the right direction. + + Relates to #38335 + +commit 66530dbde01fa30ae0be0defe708e87c462cb49e +Author: Michael Basnight +Date: Mon Feb 4 13:23:00 2019 -0600 + + Deprecate HLRC security methods (#37883) + + This commit deprecates the few methods that had their parameters + reordered to facilitate the move from EmptyResponse to boolean. This + commit also readds the boolean based methods with the proper + signatures. + + Relates #37540 + Relates #36938 + +commit be1bb0ec7d72a63232c99015836c9034af4e016e +Author: Shaunak Kashyap +Date: Mon Feb 4 10:58:03 2019 -0800 + + Remove types from Monitoring plugin "backend" code (#37745) + + This PR removes the use of document types from the monitoring exporters and template + watches setup code. + + It does not remove the notion of types from the monitoring bulk API endpoint "front end" code as that code will eventually just go away in 8.0 and be replaced with Beats as collectors/shippers directly to the monitoring cluster. + +commit ab1150378befec03b1e48722bc2dc416f2f3b329 +Author: Zachary Tong +Date: Mon Feb 4 13:47:04 2019 -0500 + + Add Composite to AggregationBuilders (#38207) + +commit 2c1eab2b8a65663c17bc86df32e209e39757ff24 +Author: David Turner +Date: Mon Feb 4 17:44:00 2019 +0000 + + Clarify slow cluster-state log messages (#38302) + + The message `... took [31s] above the warn threshold of 30s` suggests + incorrectly that the task took 61 seconds. This commit adds the clarifying + words `which is`. + +commit ac07386eabe9ff535c03b77d91ac76784f835478 +Author: Adrien Grand +Date: Mon Feb 4 17:48:08 2019 +0100 + + Ignore the `>test-mute` label when generating release notes. (#38307) + + Muted tests are irrelevant for release notes. + +commit f872c721ac2c9c4b6a168366b62f62c1258ab574 +Author: Gordon Brown +Date: Mon Feb 4 09:43:28 2019 -0700 + + Run Node deprecation checks locally (#38065) (#38250) + + At times, we need to check for usage of deprecated settings in settings + which should not be returned by the NodeInfo API. This commit changes + the deprecation info API to run all node checks locally so that these + settings can be checked without exposing them via any externally + accessible API. + +commit 1899658a38f587d7c7d1a95b9a1fb781439f4401 +Author: Christoph Büscher +Date: Mon Feb 4 17:13:54 2019 +0100 + + Mute ClusterClientIT#testClusterHealthYellowSpecificIndex (#38343) + +commit 7bc8bc960568735a1116a34c34cae944e714ed81 +Author: Andrey Ershov +Date: Mon Feb 4 16:36:04 2019 +0100 + + ensureGreen (#38324) + +commit 625d37a26a5bf4eac2dc9a7dd3da9cd327f0571d +Author: Jason Tedor +Date: Mon Feb 4 10:35:29 2019 -0500 + + Introduce retention lease background sync (#38262) + + This commit introduces a background sync for retention leases. The idea + here is that we do a heavyweight sync when adding a new retention lease, + and then periodically we want to background sync any retention lease + renewals to the replicas. As long as the background sync interval is + significantly lower than the extended lifetime of a retention lease, it + is okay if from time to time a replica misses a sync (it will still have + an older version of the lease that is retaining more data as we assume + that renewals do not decrease the retaining sequence number). There are + two follow-ups that will come after this commit. The first is to address + the fact that we have not adapted the should periodically flush logic to + possibly flush the retention leases. We want to do something like flush + if we have not flushed in the last five minutes and there are renewed + retention leases since the last time that we flushed. An additional + follow-up will remove the syncing of retention leases when a retention + lease expires. Today this sync could be invoked in the background by a + merge operation. Rather, we will move the syncing of retention lease + expiration to be done under the background sync. The background sync + will use the heavyweight sync (write action) if a lease has expired, and + will use the lightweight background sync (replication action) otherwise. + +commit 5ee7232379d5e06bcbc3dff571ac9da9af947f10 +Author: Christoph Büscher +Date: Mon Feb 4 16:10:06 2019 +0100 + + Mute SpecificMasterNodesIT#testElectOnlyBetweenMasterNodes (#38334) + +commit 715e581378276cfcec83e12e6f99aa44e8a5aed6 +Author: Christoph Büscher +Date: Mon Feb 4 15:46:19 2019 +0100 + + Mute DedicatedClusterSnapshotRestoreIT#testRestoreShrinkIndex (#38330) + +commit f19fdcd4910e8313f9dac6309373afef221b690d +Author: Lee Hinman +Date: Mon Feb 4 07:40:59 2019 -0700 + + Re-enable accounting breaker check in InternalTestCluster (#38131) + + Relates to #30290 + + The intent for this is to see whether this failure still happens, and if so, provide more up-to-date logs for analysis. + +commit fb6a176caf37fbd5cf715a7c77dbc5c516a0fb0b +Author: David Roberts +Date: Mon Feb 4 14:32:35 2019 +0000 + + [ML] Add explanation so far to file structure finder exceptions (#38191) + + The explanation so far can be invaluable for troubleshooting + as incorrect decisions made early on in the structure analysis + can result in seemingly crazy decisions or timeouts later on. + + Relates elastic/kibana#29821 + +commit e49b593c815cf31bf904370ad42f0927cb5ee1fd +Author: Boaz Leskes +Date: Mon Feb 4 15:25:41 2019 +0100 + + Move TokenService to seqno powered cas (#38311) + + Relates #37872 + Relates #10708 + +commit ece8c659c5c5b34ae5c291f262ab46156abbc520 +Author: Yannick Welsch +Date: Mon Feb 4 15:11:12 2019 +0100 + + Decrease leader and follower check timeout (#38298) + + Reduces the leader and follower check timeout to 3 * 10 = 30s instead of 3 * 30 = 90s, with 30s still + being a very long time for a node to be completely unresponsive. + +commit 9b64558efb71f1564cd97c02e427a52f521cef8c +Author: Przemyslaw Gomulka +Date: Mon Feb 4 15:08:31 2019 +0100 + + Migrating from joda to java.time. Watcher plugin (#35809) + + part of the migrating joda time work. Migrating watcher plugin to use JDK's java-time + + refers #27330 + +commit d975f939679df998a1e9abd37620afb88a910ef7 +Author: Daniel Mitterdorfer +Date: Mon Feb 4 15:03:31 2019 +0100 + + Use stricter timer in DeadHostStateTests (#38301) + + With this commit we add a monotonically strict timer to ensure time is + advancing even if the timer is called in a tight loop in tests. We also + relax a condition in a similar test so it only checks that time is not + moving backwards. + + Closes #33747 + +commit 85b4bfe3ff119b3b7fa48631c6fc4a0bf9e1816d +Author: Przemyslaw Gomulka +Date: Mon Feb 4 14:47:08 2019 +0100 + + Core: Migrating from joda to java.time. Monitoring plugin (#36297) + + monitoring plugin migration from joda to java.time + + refers #27330 + +commit 7ed3e6e07e36a7978c20a3fb85263b8a319997d7 +Author: Christoph Büscher +Date: Mon Feb 4 11:38:01 2019 +0100 + + Mute MlMigrationFullClusterRestartIT#testMigration (#38315) + +commit 87f357912563f3b40a12d6c908eabd309d0e0fc1 +Author: Alexander Reelsen +Date: Mon Feb 4 11:31:16 2019 +0100 + + Add nanosecond field mapper (#37755) + + This adds a dedicated field mapper that supports nanosecond resolution - + at the price of a reduced date range. + + When using the date field mapper, the time is stored as milliseconds since the epoch + in a long in lucene. This field mapper stores the time in nanoseconds + since the epoch - which means its range is much smaller, ranging roughly from + 1970 to 2262. + + Note that aggregations will still be in milliseconds. + However docvalue fields will have full nanosecond resolution + + Relates #27330 + +commit 15510da2af1404cb7eed61bbd7f1ff449686d2a5 +Author: Christoph Büscher +Date: Mon Feb 4 10:41:35 2019 +0100 + + Mute SharedClusterSnapshotRestoreIT#testAbortedSnapshotDuringInitDoesNotStart (#38304) + +commit ff13a431445a60678e095fa54410d2ba55fa10ac +Author: Boaz Leskes +Date: Mon Feb 4 10:41:08 2019 +0100 + + Move ML Optimistic Concurrency Control to Seq No (#38278) + + This commit moves the usage of internal versioning for CAS operations to use sequence numbers and primary terms + + Relates to #36148 + Relates to #10708 + +commit 1d82a6d9f9c1d64c1a07dac139d20a85c3139ce6 +Author: David Turner +Date: Mon Feb 4 08:52:08 2019 +0000 + + Deprecate unused Zen1 settings (#38289) + + Today the following settings in the `discovery.zen` namespace are still used: + + - `discovery.zen.no_master_block` + - `discovery.zen.hosts_provider` + - `discovery.zen.ping.unicast.concurrent_connects` + - `discovery.zen.ping.unicast.hosts.resolve_timeout` + - `discovery.zen.ping.unicast.hosts` + + This commit deprecates all other settings in this namespace so that they can be + removed in the next major version. + +commit 4561f425db6193dfe5cd898cc404cfed8127e339 +Author: Armin Braun +Date: Mon Feb 4 09:06:39 2019 +0100 + + Remove Redundandant Loop in SnapshotShardsService (#38283) + + * This was a merge mistake on my end I think, obviously we only need to loop over the shards once not twice here to find those that we missed in INIT state + +commit d58e899d45fcc3ac8e8c58e110b6bbe0a5f2e065 +Author: Alpar Torok +Date: Mon Feb 4 10:05:04 2019 +0200 + + Remove empty service files (#38192) + +commit 0164acb0a7edc8bbf4d07fc44c5a084bd2b04011 +Author: Tim Vernum +Date: Mon Feb 4 17:27:41 2019 +1100 + + Cleanup construction of interceptors (#38294) + + It would be beneficial to apply some of the request interceptors even + when features are disabled. This change reworks the way we build that + list so that the interceptors we always want to use are constructed + outside of the settings check. + +commit 75f0750ff7495073557a0dd6bc274fb1a26a1d12 +Author: Costin Leau +Date: Sun Feb 3 22:32:16 2019 +0200 + + SQL: Remove exceptions from Analyzer (#38260) + + Instead of throwing an exception, use an unresolved attribute to pass + the message to the Verifier. + Additionally improve the parser to save the extended source for the + Aggregate and OrderBy. + + Close #38208 + +commit a088155f4d6ad78310e4728729db989cbdd005e0 +Author: Costin Leau +Date: Sun Feb 3 22:31:35 2019 +0200 + + SQL: Move metrics tracking inside PlanExecutor (#38259) + + Move metrics in one place, from the transport layer inside the + PlanExecutor + Remove unused class + + Close #38258 + +commit 820029522b0e6774e64a7816a2ddd95031d9acad +Author: Christoph Büscher +Date: Sun Feb 3 19:52:53 2019 +0100 + + Mute DateProcessorTests#testJodaPatternLocale (#38265) + + Only fails on FIPS 8, muting this selectively. + +commit d2cc1459a35594f9d55d6c8dc42729c9c688b2eb +Author: Jason Tedor +Date: Sun Feb 3 12:54:31 2019 -0500 + + Fix ordering problem in add or renew lease test (#38280) + + We have to set the primary term before we add a retention lease, + otherwise we can not assert the correct primary term. + +commit 6ca7a913ea6b205ddadb06d0c9e751f57135eef4 +Author: Christoph Büscher +Date: Sun Feb 3 12:54:13 2019 +0100 + + Mute ReplicationTrackerRetentionLeaseTests#testAddOrRenewRetentionLease (#38275) + +commit 3c1544d2593c375091c9fd6f772c4079730b19cd +Author: Albert Zaharovits +Date: Sun Feb 3 10:34:17 2019 +0200 + + Fix NPE in Logfile Audit Filter (#38120) + + The culprit in #38097 is an `IndicesRequest` that has no indices, + but instead of `request.indices()` returning `null` or `String[0]` + it returned `String[] {null}` . This tripped the audit filter. + + I have addressed this in two ways: + 1. `request.indices()` returning `String[] {null}` is treated as `null` + or `String[0]`, i.e. no indices + 2. `null` values among the roles and indices lists, which are + unexpected, will never again stumble the audit filter; `null` values + are treated as special values that will not match any policy, + i.e. their events will always be printed. + + Closes #38097 + +commit 89d7c57bd91d8f03b9a9e589d93aae3954313ad1 +Author: Armin Braun +Date: Sun Feb 3 08:48:15 2019 +0100 + + Fix Incorrect Transport Response Handler Type (#38264) + + * Fix Incorrect Transport Response Handler Type + * The response type here is not empty and was always wrong but this only became visible now that 0a604e3b2496777168bb541815a8526e463ee15f was introduced + * As a result of 0a604e3b2496777168bb541815a8526e463ee15f we started actually handling the response + of this request and logging/handling exceptions before that we simply dropped the classcast exception here quietly using the empty response handler + * fix busy assert not handling `Exception` + * Closes #38226 + * Closes #38256 + +commit 0861dc358153492b42d84432bb14d3030143baeb +Author: Nhat Nguyen +Date: Sat Feb 2 20:02:21 2019 -0500 + + Mute testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData (#38268) + + Tracked at #38267 + +commit 6968f0925b3304a66516916cf69d4bcb9749ea30 +Author: Andrei Stefan +Date: Sat Feb 2 22:05:47 2019 +0200 + + SQL: Generate relevant error message when grouping functions are not used in GROUP BY (#38017) + + * Add checks for Grouping functions restriction to be placed inside GROUP BY + * Fixed bug where GROUP BY HISTOGRAM (not using alias) wasn't recognized + properly in the Verifier due to functions equality not working correctly. + +commit 75abb5b8a6b7b3cb9e54cd40c4afaa635fda2e87 +Author: Nhat Nguyen +Date: Sat Feb 2 12:09:14 2019 -0500 + + Adapt LLRest warning exception in FullClusterRestartIT (#38253) + + We now throw a WarningFailureException instead of ResponseException if + there's any warning in a response. This change leads to the failures of + testSnapshotRestore in the BWC builds for the last two days. + + Relates #37247 + +commit 50cdc618747b9fa42bc277bf36975d985a2f64c3 +Author: Christoph Büscher +Date: Sat Feb 2 13:46:29 2019 +0100 + + Mute DedicatedClusterSnapshotRestoreIT#testRestoreShrinkIndex (#38257) + +commit c3110624763bcdec584269473a5dda28aa18c45e +Author: David Turner +Date: Sat Feb 2 07:54:56 2019 +0000 + + Add CoordinatorTests for empty unicast hosts list (#38209) + + Today we have DiscoveryDisruptionIT tests for checking that discovery can still + work once the cluster has formed, even if the cluster is misconfigured and only + has a single master-eligible node in its unicast hosts list. In fact with Zen2 + we can go one better: we do not need any nodes in the unicast hosts list, + because nodes also use the contents of the last-committed cluster state for + discovery. Additionally, the DiscoveryDisruptionIT tests were failing due to + the overenthusiastic fault-detection timeouts. + + This commit replaces these tests with deterministic `CoordinatorTests` that + verify the same behaviour. It also removes some duplication by extracting a + test method called `testFollowerCheckerAfterMasterReelection()` + + Closes #37687 + +commit 80d309229245929778d117a61c8db8a7c69da0f6 +Author: Nhat Nguyen +Date: Sat Feb 2 02:38:53 2019 -0500 + + Fix primary term in testAddOrRenewRetentionLease (#38239) + + We should increase primary term before renewing leases; otherwise, the + term of the latest RetentionLeases will be lower than the current term. + + Relates #37951 + +commit 1ec04dff43576f05e395e020705320b41b3c4a2e +Author: Nhat Nguyen +Date: Sat Feb 2 02:37:37 2019 -0500 + + FIx testReplicaIgnoresOlderRetentionLeasesVersion (#38246) + + If the innerLength is 0, the version won't be increased; then there will + be two RetentionLeases with the same term and version, but their leases + are different. + + Relates #37951 + Closes #38245 + +commit 475a045192e76b911d4e371bb921d0ca66155a46 +Author: Gordon Brown +Date: Fri Feb 1 21:00:58 2019 -0700 + + Mute tests in SSLConfigurationReloaderTests (#38248) + + Specifically `testReloadingTrustStore` and `testReloadingPEMTrustConfig` + +commit 7a1e89c7ed819b9342ebc33792d5776e24c72e8a +Author: Gordon Brown +Date: Fri Feb 1 20:46:12 2019 -0700 + + Ensure ILM policies run safely on leader indices (#38140) + + Adds a Step to the Shrink and Delete actions which prevents those + actions from running on a leader index - all follower indices must first + unfollow the leader index before these actions can run. This prevents + the loss of history before follower indices are ready, which might + otherwise result in the loss of data. + +commit 8bee5b8e0663f64b5fdaf274cd85d23e7bc83f9b +Author: Nhat Nguyen +Date: Fri Feb 1 21:27:10 2019 -0500 + + Mute testAddOrRenewRetentionLease (#38240) + + Relates #38239 + +commit f6e06a2b19fce03ac0c923d38a475e2a3b7fe9a3 +Author: Boaz Leskes +Date: Sat Feb 2 02:37:16 2019 +0100 + + Adapt minimum versions for seq# powered operations in Watch related requests and UpdateRequest (#38231) + + After backporting #37977, #37857 and #37872 + +commit 783c9ed372428180295c52a2a589bba0fc53afd1 +Author: Costin Leau +Date: Sat Feb 2 01:38:25 2019 +0200 + + SQL: Allow sorting of groups by aggregates (#38042) + + Introduce client-side sorting of groups based on aggregate + functions. To allow this, the Analyzer has been extended to push down + to underlying Aggregate, aggregate function and the Querier has been + extended to identify the case and consume the results in order and sort + them based on the given columns. + The underlying QueryContainer has been slightly modified to allow a view + of the underlying values being extracted as the columns used for sorting + might not be requested by the user. + + The PR also adds minor tweaks, mainly related to tree output. + + Close #35118 + +commit 630889baec86ab8edfd554566034155a5af7f713 +Author: Jack Conradson +Date: Fri Feb 1 15:10:59 2019 -0800 + + Remove extraneous test from Painless lambda tests (#38111) + + This test has been awaiting a fix that isn't currently relevant because incoming + lambda parameters are read-only. If this ever changes a new set of tests can + be added that are up-to-date. + +commit f181e170387ca50727b750605fad51a3e85fed3a +Author: Jason Tedor +Date: Fri Feb 1 17:19:19 2019 -0500 + + Introduce retention leases versioning (#37951) + + Because concurrent sync requests from a primary to its replicas could be + in flight, it can be the case that an older retention leases collection + arrives and is processed on the replica after a newer retention leases + collection has arrived and been processed. Without a defense, in this + case the replica would overwrite the newer retention leases with the + older retention leases. This commit addresses this issue by introducing + a versioning scheme to retention leases. This versioning scheme is used + to resolve out-of-order processing on the replica. We persist this + version into Lucene and restore it on recovery. The encoding of + retention leases is starting to get a little ugly. We can consider + addressing this in a follow-up. + +commit 78a65c340d195e99a533ce57b6c263d1bb73dd31 +Author: Ioannis Kakavas +Date: Fri Feb 1 23:56:35 2019 +0200 + + Correctly disable tests for FIPS JVMs (#38214) + + Replace assertFalse with assumeFalse + + Resolves: #38212 + +commit 9c39dea7ae0c623a37313e399f4e81a534b10279 +Author: Nhat Nguyen +Date: Fri Feb 1 16:24:02 2019 -0500 + + AwaitsFix testAbortedSnapshotDuringInitDoesNotStart (#38227) + + Tracked at #38226 + +commit bae656dceafb57278cd5ad1f0f933c4e6c6cc124 +Author: Tal Levy +Date: Fri Feb 1 13:16:34 2019 -0800 + + Preserve ILM operation mode when creating new lifecycles (#38134) + + There was a bug where creating a new policy would start + the ILM service, even if it was stopped. This change ensures + that there is no change to the existing operation mode + +commit 3ecdfe106076d9820ca86ee4369b69e708ec9088 +Author: Nhat Nguyen +Date: Fri Feb 1 15:44:39 2019 -0500 + + Enable trace log in FollowerFailOverIT (#38148) + + This suite still fails one per week sometimes with a worrying assertion. + Sadly we are still unable to find the actual source. + + Expected: + but: was + + This change enables trace log in the suite so we will have a better + picture if this fails again. + + Relates #3333 + +commit 03a1d2107069123e6a7f8319e4cccffaea7eb9f2 +Author: Armin Braun +Date: Fri Feb 1 20:46:14 2019 +0100 + + SnapshotShardsService Simplifications (#38025) + + * Instead of replacing the `shardSnapshots` field, we mutate it, explicitly removing entries from it in only a single spot + * Decreased the amount of indirection by moving all logic for starting a snapshot's newly discovered shard tasks into `startNewShards` (saves us two maps (keyed by snapshot) and iterations over them) + +commit c2e9d13ebd63d9c45135b09583e5aa8555ec2672 +Author: Julie Tibshirani +Date: Fri Feb 1 11:44:13 2019 -0800 + + Default include_type_name to false in the yml test harness. (#38058) + + This PR removes the temporary change we made to the yml test harness in #37285 + to automatically set `include_type_name` to `true` in index creation requests + if it's not already specified. This is possible now that the vast majority of + index creation requests were updated to be typeless in #37611. A few additional + tests also needed updating here. + + Additionally, this PR updates the test harness to set `include_type_name` to + `false` in index creation requests when communicating with 6.x nodes. This + mirrors the logic added in #37611 to allow for typeless document write requests + in test set-up code. With this update in place, we can remove many references + to `include_type_name: false` from the yml tests. + +commit 9350da98a76a15304b45e11ba97ae34ccadc74e4 +Author: Boaz Leskes +Date: Fri Feb 1 20:28:08 2019 +0100 + + Disable bwc preparing to backport of#37977, #37857 and #37872 (#38126) + +commit a70f54fc775284a7595360401158605c4590678b +Author: Benjamin Trent +Date: Fri Feb 1 12:45:28 2019 -0600 + + Adding ml_settings entry to HLRC and Docs for deprecation_info (#38118) + +commit f64b20383ed7f86c400a63be3e23145ae4440843 +Author: Nhat Nguyen +Date: Fri Feb 1 13:31:17 2019 -0500 + + Replace awaitBusy with assertBusy in atLeastDocsIndexed (#38190) + + Unlike assertBusy, awaitBusy does not retry if the code-block throws an + AssertionError. A refresh in atLeastDocsIndexed can fail because we call + this method while we are closing some node in FollowerFailOverIT. + +commit ee57420de6e25f83d412d8d4cf204eea0012f952 +Author: Luca Cavanna +Date: Fri Feb 1 19:23:13 2019 +0100 + + Adjust SearchRequest version checks (#38181) + + The finalReduce flag is now supported on 6.x too, hence we need to update the version checks in master. + +commit 70235838d1db84295c64eca1651c10fef833aa50 +Author: Nhat Nguyen +Date: Fri Feb 1 12:50:07 2019 -0500 + + AwaitsFix testClientSucceedsWithVerificationDisabled (#38213) + + Tracked at #38212 + +commit 04dc41b99e8fa7b85e7f5cf8e0655f1459a4c943 +Author: Andrey Ershov +Date: Fri Feb 1 18:18:11 2019 +0100 + + Zen2ify RareClusterStateIT (#38184) + + In Zen 1 there are commit timeout and publish timeout and these + settings could be changed on-the-fly. + + In Zen 2, there is only commit timeout and this setting is static. + RareClusterStateIT is actively using these settings and the fact, they + are dynamic. + + This commit adds cancelCommitedPublication method to Coordinator to + be used by tests. This method will cancel current committed publication + if there is any. + When there is BlockClusterStateProcessing on the non-master node, the + publication will be accepted and committed, but not yet applied. So we + can use the method above to cancel it. + + Also, this commit replaces callback + AtomicReference with ActionFuture, + which makes test code easier to read. + +commit 5db305023df7c7d729d4d175fb4a775a0d851ff9 +Author: Benjamin Trent +Date: Fri Feb 1 11:16:35 2019 -0600 + + ML: Fix error race condition on stop _all datafeeds and close _all jobs (#38113) + + * ML: Ignore when task is not found for _all + + * Addressing PR comments + + * Update TransportStopDatafeedAction.java + +commit f5f3cb8f4c9f63401f8aec7262ac85797b661cec +Author: Nhat Nguyen +Date: Fri Feb 1 12:00:43 2019 -0500 + + AwaitsFix PUT mapping with _doc on an index that has types (#38204) + + Tracked at #38202 + +commit cc7c42d7e2e221ceacbb368c027aae60f1b761eb +Author: Shaunak Kashyap +Date: Fri Feb 1 08:56:34 2019 -0800 + + Allow built-in monitoring_user role to call GET _xpack API (#38060) + + This PR adds the `monitor/xpack/info` cluster-level privilege to the built-in `monitoring_user` role. + + This privilege is required for the Monitoring UI to call the `GET _xpack API` on the Monitoring Cluster. It needs to do this in order to determine the license of the Monitoring Cluster, which further determines whether Cluster Alerts are shown to the user or not. + + Resolves #37970. + +commit 603cdf40f182ac876414ae2b910f3ddca90ab667 +Author: Nick Knize +Date: Fri Feb 1 10:41:41 2019 -0600 + + Update geo_shape docs to include unsupported features (#38138) + + There are a two major features that are not yet supported by BKD Backed geo_shape: MultiPoint queries, and CONTAINS relation. It is important we are explicitly clear in the documentation that using the new approach may not work for users that depend on these features. This commit adds an IMPORTANT NOTE section to geo_shape docs that explicitly highlights these missing features and what should be done if they are an absolute necessity. + +commit 1fa413a16df76fef7c17c44e5fa7e547edd56a55 +Author: David Roberts +Date: Fri Feb 1 15:36:04 2019 +0000 + + [ML] Remove "8" prefixes from file structure finder timestamp formats (#38016) + + In 7.x Java timestamp formats are the default timestamp format and + there is no need to prefix them with "8". (The "8" prefix was used + in 6.7 to distinguish Java timestamp formats from Joda timestamp + formats.) + + This change removes the "8" prefixes from timestamp formats in the + output of the ML file structure finder. + +commit 5c58c2508e76347e04a8740e9e88683f5903ee89 +Author: Luca Cavanna +Date: Fri Feb 1 16:34:24 2019 +0100 + + Disable bwc tests while backporting #38104 (#38182) + + Relates to #38180 + +commit 2ca22209cd52f09e5f352e147e61000f1a79e519 +Author: Jay Modi +Date: Fri Feb 1 08:34:11 2019 -0700 + + Enable TLSv1.3 by default for JDKs with support (#38103) + + This commit enables the use of TLSv1.3 with security by enabling us to + properly map `TLSv1.3` in the supported protocols setting to the + algorithm for a SSLContext. Additionally, we also enable TLSv1.3 by + default on JDKs that support it. + + An issue was uncovered with the MockWebServer when TLSv1.3 is used that + ultimately winds up in an endless loop when the client does not trust + the server's certificate. Due to this, SSLConfigurationReloaderTests + has been pinned to TLSv1.2. + + Closes #32276 + +commit 025bf2840528bb25b82cbf2542c7a545728bcbf9 +Author: Yannick Welsch +Date: Fri Feb 1 16:02:37 2019 +0100 + + Fix _host based require filters (#38173) + + Using index.routing.allocation.require._host does not correctly work because the boolean logic in + filter matching is broken (DiscoveryNodeFilters.match(...) will return false) when + opType ==OpType.AND + +commit da6269b456d5a3cf0b75d184d0b2148b8520e2b4 +Author: Tanguy Leroux +Date: Fri Feb 1 15:59:11 2019 +0100 + + RestoreService should update primary terms when restoring shards of existing indices (#38177) + + When restoring shards of existing indices, the RestoreService also + restores the values of primary terms stored in the snapshot index + metadata. The primary terms are not updated and could potentially + conflict with current index primary terms if the restored primary terms + are lower than the existing ones. + + This situation is likely to happen with replicated closed indices + (because primary terms are increased when the index is transitioning + from open to closed state, and the snapshotted primary terms are the + one at the time the index was opened) (see #38024) and maybe also + with CCR. + + This commit changes the RestoreService so that it updates the primary + terms using the maximum value between the snapshotted values and + the existing values. + + Related to #33888 + +commit c1c4abae10266f82e7ec2a34ed56c60e078023b7 +Author: Desmond Vehar +Date: Fri Feb 1 06:53:50 2019 -0800 + + Throw if two inner_hits have the same name (#37645) + + This change throws an error if two inner_hits have the same name + + Closes #37584 + +commit 35ed137684778ae80363576318d2919a32dce79e +Author: Alexander Reelsen +Date: Fri Feb 1 15:42:56 2019 +0100 + + Ensure joda compatibility in custom date formats (#38171) + + If custom date formats are used, there may be combinations that the new + performat DateFormatters.from() method has not covered yet. This adds a + few such corner cases and ensures the tests are correctly commented + out. + +commit 66e4fb4fb6e1b4cd1224409cb2f0541bcfb1900f +Author: Jim Ferenczi +Date: Fri Feb 1 15:32:19 2019 +0100 + + Do not compute cardinality if the `terms` execution mode does not use `global_ordinals` (#38169) + + In #38158 we ensured that global ordinals are not loaded when another execution hint is explicitly set on the source. This change is a follow up that addresses a comment + https://github.com/elastic/elasticsearch/pull/38158/files/dd6043c1c019974fe1c58810384b89e30cd8b89b#r252984782 added after the merge. + +commit 2e475d63f7ca5d2354c1f603539763adae85f49e +Author: Nhat Nguyen +Date: Fri Feb 1 09:30:03 2019 -0500 + + Do not set timeout for IndexRequests in GatewayIndexStateIT (#38147) + + CI might not be fast enough to publish a dynamic mapping update within 100ms. + +commit c1270e97b004a7fbeeb02d3265029222f6ac0ab9 +Author: Andrey Ershov +Date: Fri Feb 1 15:24:08 2019 +0100 + + Zen2ify testMasterFailoverDuringIndexingWithMappingChanges (#38178) + + In Zen2 cluster bootstrap is required and some parameters are + called differently in Zen2. + +commit 19dccf8f3e75aa26fb0b451d48939e56daf7cc92 +Author: Marios Trivyzas +Date: Fri Feb 1 16:13:51 2019 +0200 + + SQL: [Docs] Add limitation for aggregate functions on scalars (#38186) + + Currently aggregate functions can operate only directly on fields. + They cannot be used on top of scalar functions as painless scripting + is currently not supported. + +commit bda591453cc4cfc076eb2bccc4b96877f27f393c +Author: Andrey Ershov +Date: Fri Feb 1 14:53:55 2019 +0100 + + Add elasticsearch-node detach-cluster command (#37979) + + This commit adds the second part of `elasticsearch-node` tool - + `detach-cluster` command in addition to `unsafe-bootstrap` command. + Also, this commit changes the semantics of `unsafe-bootstrap`, now + `unsafe-bootstrap` changes clusterUUID. + So the algorithm of running `elasticsearch-node` tool is the following: + 1) Stop all nodes in the cluster. + 2) Pick master-eligible node with the highest (term, version) pair and + run the `unsafe-bootstrap` command on it. If there are no survived + master-eligible nodes - skip this step. + 3) Run `detach-cluster` command on the remaining survived nodes. + + Detach cluster makes the following changes to the node metadata: + 1) Sets clusterUUID committed to false. + 2) Sets currentTerm and term to 0. + 3) Removes voting tombstones and sets voting configurations to special + constant MUST_JOIN_ELECTED_MASTER, that prevents initial cluster + bootstrap. + + `ElasticsearchNodeCommand` base abstract class is introduced, because + `UnsafeBootstrapMasterCommand` and `DetachClusterCommand` have a lot in + common. + Also, this commit adds "ordinal" parameter to both commands, because it's + impossible to write IT otherwise. + For MUST_JOIN_ELECTED_MASTER case special handling is introduced in + `ClusterFormationFailureHelper`. + Tests for both commands reside in `ElasticsearchNodeCommandIT` (renamed + from `UnsafeBootstrapMasterIT`). + +commit 979e5576e5fcabec7cde63cca845dcd43b06584d +Author: Alexander Reelsen +Date: Fri Feb 1 14:48:37 2019 +0100 + + Add tests for fractional epoch parsing (#38162) + + Fractional epoch parsing is supported, the tests we used were edge cases + that did not make sense. This adds tests to properly check for this. + +commit 2229e7231e1ddb139b52cc9d3cd770aeebe64474 +Author: Adrien Grand +Date: Fri Feb 1 13:55:51 2019 +0100 + + Enable bw tests for #37871 and #38032. (#38167) + + Mixed-version clusters tests had been disabled initially since they wouldn't + work until the functionality would be backported. + +commit 029e4b6278c91670a509bd9ce35f0081e51513cd +Author: Tanguy Leroux +Date: Fri Feb 1 12:58:46 2019 +0100 + + Clear send behavior rule in CloseWhileRelocatingShardsIT (#38159) + + The current CloseWhileRelocatingShardsIT test adds some "send behavior" + rule to a target node's mocked transport service in order to detect when shard + relocating are started. These rules are never cleared and prevent the test to + complete normally after the rebalance is re-enabled again. + + This commit changes the test so that rules are cleared and most verifications + are done before the rebalance is reenabled again. + + Closes #38090 + +commit ce469cfda57dcdfac8e21a62d1057df7b360534d +Author: Yannick Welsch +Date: Fri Feb 1 12:51:38 2019 +0100 + + Fix testCorruptedIndex (#38161) + + Folks at the Lucene project do not seem to be interested in classifying corruptions and + distinguishing them from file-system exceptions (see https://issues.apache.org/jira/browse/LUCENE-8525), + so we'll just cop out as well. + + Closes #34322 + +commit e18cac3659a82b60884564ec78d7c36b6eb0817a +Author: Luca Cavanna +Date: Fri Feb 1 12:11:42 2019 +0100 + + Add finalReduce flag to SearchRequest (#38104) + + With #37000 we made sure that fnial reduction is automatically disabled + whenever a localClusterAlias is provided with a SearchRequest. + + While working on #37838, we found a scenario where we do need to set a + localClusterAlias yet we would like to perform a final reduction in the + remote cluster: when searching on a single remote cluster. + + Relates to #32125 + + This commit adds support for a separate finalReduce flag to + SearchRequest and makes use of it in TransportSearchAction in case we + are searching against a single remote cluster. + + This also makes sure that num_reduce_phases is correct when searching + against a single remote cluster: it makes little sense to return + `num_reduce_phases` set to `2`, which looks especially weird in case + the search was performed against a single remote shard. We should + perform one reduction phase only in this case and `num_reduce_phases` + should reflect that. + + * line length + +commit 6fa93ca49387d5f3ae3208ed64623a0c319d570e +Author: Jim Ferenczi +Date: Fri Feb 1 11:41:40 2019 +0100 + + Forbid negative field boosts in analyzed queries (#37930) + + This change forbids negative field boost in the `query_string`, `simple_query_string` + and `multi_match` queries. + Negative boosts are not allowed in Lucene 8 (scores must be positive). + The backport of this change to 6x will turn the error into a deprecation warning + in order to raise the awareness of this breaking change in 7.0. + + Closes #33309 + +commit 57b1d245e8e9e766cb708088dd78fbc691557dfc +Author: Jim Ferenczi +Date: Fri Feb 1 11:41:17 2019 +0100 + + Remove AtomiFieldData#getLegacyFieldValues (#38087) + + This function is unused now that we format the docvalue fields with the default + formatter on the field (#30831) + +commit bfd618cf8324ef3d1f67f304286bf468eb84ad8e +Author: Andrey Ershov +Date: Fri Feb 1 11:34:31 2019 +0100 + + Universal cluster bootstrap method for tests with autoMinMasterNodes=false (#38038) + + Currently, there are a few tests that use autoMinMasterNodes=false and + hence override addExtraClusterBootstrapSettings, mostly this is 10-30 + lines of codes that are copy-pasted from class to class. + + This PR introduces `InternalTestCluster.setBootstrapMasterNodeIndex` + which is suitable for all classes and copy-paste could be removed. + + Removing code is always a good thing! + +commit 0e6a7c20a1730156756fad33d5f3297958a39db8 +Author: Tanguy Leroux +Date: Fri Feb 1 11:16:54 2019 +0100 + + Fix FullClusterRestartIT.testHistoryUUIDIsAdded (#38098) + + This test failed once because the index wasn't fully ready + (ie, engine opened). This commit changes the test so that + it waits for the index to be green before checking the history + UUID. + + Closes #34452 + +commit 6c5a7387af086381da34db6103fc711dd8350b14 +Author: Alexander Reelsen +Date: Fri Feb 1 10:15:18 2019 +0100 + + Replace joda time in ingest-common module (#38088) + + This commit fully replaces any remaining joda time time classes with + java time implementations. + + Relates #27330 + +commit d417997aca4a6e91e6bc0f0c8141af0e29b7255a +Author: Alpar Torok +Date: Fri Feb 1 10:47:54 2019 +0200 + + Fix eclipse config for ssl-config (#38096) + +commit b7308aa03cdf87d12a0ff781b2bbb7b6222fcdad +Author: Jim Ferenczi +Date: Fri Feb 1 09:35:46 2019 +0100 + + Don't load global ordinals with the `map` execution_hint (#37833) + + The terms aggregator loads the global ordinals to retrieve the cardinality of the field to aggregate on. This information is then used to select the strategy to use for the aggregation (breadth_first or depth_first). However this should be avoided if the execution_hint is explicitly set to map since this mode doesn't really need the global ordinals. Since we still need the cardinality of the field this change picks the maximum cardinality in the segments as an estimation of the total cardinality to select the strategy to use (breadth_first or depth_first). This estimation is only used if the execution hint is set to map, otherwise the global ordinals are still used to retrieve the accurate cardinality. + + Closes #37705 + +commit 23f00e367673bec2637deb52d95e9c6bee3d4963 +Author: David Turner +Date: Fri Feb 1 08:10:49 2019 +0000 + + Relax fault detector in some disruption tests (#38101) + + Today we use `AbstractDisruptionTestCase` to test the behaviour of things like + master elections in the presence of cluster disruptions. These tests have + rather enthusiastic fault detection settings, detecting a fault if a single + ping fails, with a one-second timeout. Furthermore there are some tests that + assert the identity of the master remains unchanged during some disruption, and + these assertions fail rather often thanks to the overly sensitive fault + detector. + + However in a number of these tests the fault detector need not be this + sensitive. This commit moves some such tests into their own test suite and uses + more sensible fault-detection settings to avoid the kind of master instability + that is causing CI failures. + + Closes #37699 + +commit c02cd3e2fdc703ade351f43f5e733279b324e9fb +Author: Alexander Reelsen +Date: Fri Feb 1 09:03:48 2019 +0100 + + Fix java time epoch date formatters (#37829) + + The self written epoch date formatters were not properly able to format + an Instant to a string due to a misconfiguration. + + This fix also removes a until now existing runtime behaviour under java + 8 regarding the names of the aggregation buckets, which are now the same + as before and have been under java 11. + +commit 859e2f5bc8739e3a3afa8e2f94c78906b5f5a4d0 +Author: Yannick Welsch +Date: Fri Feb 1 08:57:26 2019 +0100 + + Adapt timeouts in UpdateMappingIntegrationIT + + Relates to #37263 and possibly #36916 + +commit d83c748417da334010d0659ed2f5b2c9052fb7d7 +Author: Adrien Grand +Date: Fri Feb 1 08:35:29 2019 +0100 + + Fix test bug in DynamicMappingsIT. (#37906) + + Closes #37898 + +commit 275857857099f711276e4399f276cf1e95912613 +Author: Przemyslaw Gomulka +Date: Fri Feb 1 08:12:12 2019 +0100 + + Trim the JSON source in indexing slow logs (#38081) + + The '{' as a first character in log line is causing problems for beats when parsing plaintext logs. This can happen if the submitted document has an additional '\n' at the beginning and we are not reformatting. + Trimming the source part of a SlogLog solves that and keeps the logs readable. + + closes #38080 + +commit 6fcbd07420f1f779a58fa5d49b834964896a2ac1 +Author: Tim Vernum +Date: Fri Feb 1 17:59:13 2019 +1100 + + Remove heuristics that enable security on trial licenses (#38075) + + In 6.3 trial licenses were changed to default to security + disabled, and ee added some heuristics to detect when security should + be automatically be enabled if `xpack.security.enabled` was not set. + + This change removes those heuristics, and requires that security be + explicitly enabled (via the `xpack.security.enabled` setting) for + trial licenses. + + Relates: #38009 + +commit 0a604e3b2496777168bb541815a8526e463ee15f +Author: Armin Braun +Date: Fri Feb 1 05:45:40 2019 +0100 + + Fix Two Races that Lead to Stuck Snapshots (#37686) + + * Fixes two broken spots: + 1. Master failover while deleting a snapshot that has no shards will get stuck if the new master finds the 0-shard snapshot in `INIT` when deleting + 2. Aborted shards that were never seen in `INIT` state by the `SnapshotsShardService` will not be notified as failed, leading to the snapshot staying in `ABORTED` state and never getting deleted with one or more shards stuck in `ABORTED` state + * Tried to make fixes as short as possible so we can backport to `6.x` with the least amount of risk + * Significantly extended test infrastructure to reproduce the above two issues + * Two new test runs: + 1. Reproducing the effects of node disconnects/restarts in isolation + 2. Reproducing the effects of disconnects/restarts in parallel with shard relocations and deletes + * Relates #32265 + * Closes #32348 + +commit 0d56955d39e9e5e3e66aaf8ea63a13f10606bf7e +Author: Paul Sanwald +Date: Thu Jan 31 22:00:09 2019 -0500 + + mute test, as this one is failing also per #35450 (#38132) + +commit b8b843476d387f257a50c53db586abe855ea0fb9 +Author: Nhat Nguyen +Date: Thu Jan 31 21:01:41 2019 -0500 + + Disable dynamic mapping in testSimpleGetFieldMappingsWithDefaults (#38045) + + Since #31140 we no longer require acking on the dynamic mapping of index + requests. Thus, a returned mapping from a get mapping request does not + necessarily contain the dynamic updates from the index request. This + commit replaces the dynamic mapping update with a manual put mapping. + + Relates #31140 + Closes #37928 + +commit a8ebe2a217ae74476b7e361228cbd66bca7cf8ab +Author: Nhat Nguyen +Date: Thu Jan 31 20:50:41 2019 -0500 + + Fix random params in testSoftDeletesRetentionLock (#38114) + + Since #37992 the retainingSequenceNumber is initialized with 0 + while the global checkpoint can be -1. + + Relates #37992 + +commit 291c4e7a0ce63f25a238efc34bbef8b224ac1ec9 +Author: Tim Brooks +Date: Thu Jan 31 18:02:24 2019 -0700 + + Fix file reading in ccr restore service (#38117) + + Currently we use the raw byte array length when calling the IndexInput + read call to determine how many bytes we want to read. However, due to + how BigArrays works, the array length might be longer than the reference + length. This commit fixes the issue and uses the BytesRef length when + calling read. Additionally, it expands the index follow test to index + many more documents. These documents should potentially lead to large + enough segment files to trigger scenarios where this fix matters. + +commit c67a9663afd4b9711ab4890395830da6659d1565 +Author: Lee Hinman +Date: Thu Jan 31 17:13:19 2019 -0700 + + Fix MasterServiceTests.testClusterStateUpdateLogging (#38116) + + This changes the test to not use a `CountDownlatch`, instead adding an assertion + for the final logging message and waiting until the `MockAppender` has seen it + before proceeding. + + Related to df2c06f6f30f7e23a6863a3f72fc3bdb7648885c + Resolves #23739 + +commit f3cde06a1dcbe57d1fb6641a2af8910ca4f8db6d +Author: Yuri Astrakhan +Date: Thu Jan 31 19:11:30 2019 -0500 + + geotile_grid implementation (#37842) + + Implements `geotile_grid` aggregation + + This patch refactors previous implementation https://github.com/elastic/elasticsearch/pull/30240 + + This code uses the same base classes as `geohash_grid` agg, but uses a different hashing + algorithm to allow zoom consistency. Each grid bucket is aligned to Web Mercator tiles. + +commit 6c1e9fad4707956c0bb7091c5edb893f60e1ca3f +Author: Nhat Nguyen +Date: Thu Jan 31 16:57:53 2019 -0500 + + Mute testAutoFollowing + + Tracked at #37231 + +commit a3d9ba3f4b9de2464a782eae333978411f4c9767 +Author: Pascal Christoph +Date: Thu Jan 31 22:33:13 2019 +0100 + + Log document id when MapperParsingException occurs (#37800) + + Closes #37658 + +commit 5b008a34aa3c07e37b12b415d3c22a44da491329 +Author: Jake Landis +Date: Thu Jan 31 13:54:34 2019 -0600 + + Ingest node - user agent, move device to an object (#38115) + + When the ingest node user agent parses the device field, it + will result in a string value. To match the ecs schema + this commit moves the value of the parsed device to an + object with an inner field named 'name'. There are not + any passivity concerns since this modifies an unreleased change. + + closes #38094 + relates #37329 + +commit 91b79ebed41b489eb6844ef574160bb0d07675d6 +Author: Julie Tibshirani +Date: Thu Jan 31 10:26:24 2019 -0800 + + Update 'removal of types' docs to reflect the new plan. (#38003) + +commit 237fcda2cca531ee1addebefe03594c7c3afb5a8 +Author: Nhat Nguyen +Date: Thu Jan 31 13:16:52 2019 -0500 + + Disable dynamic mapping update in testTransportBulkTasks (#38073) + + If a replica does not have a right mapping yet, we will retry the index + request on that replica; then the actual tasks is higher than the + expected tasks. Since #31140 this happens more frequently for we no + longer require acking on the dynamic mapping of index requests. + + Relates #31140 + Closes #37893 + +commit 28b5c7ce7848f8da4739bb082d4030821846b1ce +Author: Przemyslaw Gomulka +Date: Thu Jan 31 18:59:40 2019 +0100 + + Do not set up NodeAndClusterIdStateListener in test (#38110) + + When extending ESIntegTestCase are run on the same jvm, the static field in + NodeAndClusterIdConverter will throw an AlreadySet exceptions. + overriding the configuration method from Node.configureNodeAndClusterIdStateListener in the MockNode will prevent the listener registration from happening + relates #32850 + +commit be381b45256b368c105832cd1f52f3c1695d39ed +Author: Benjamin Trent +Date: Thu Jan 31 11:07:54 2019 -0600 + + ML: better handle task state race condition (#38040) + +commit 8e95780f98d9ed94f87d61af27923395ded3dae1 +Author: Nhat Nguyen +Date: Thu Jan 31 12:02:57 2019 -0500 + + Soft-deletes policy should always fetch latest leases (#37940) + + If a new retention lease is added while a primary's soft-deletes policy + is locked for peer-recovery, that lease won't be baked into the Lucene + commit. + + Relates #37165 + Relates #37375 + +commit 68ed72b92395fdbc777cad9046a6be4d695b9cd4 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 31 17:51:45 2019 +0100 + + Handle scheduler exceptions (#38014) + + Scheduler.schedule(...) would previously assume that caller handles + exception by calling get() on the returned ScheduledFuture. + schedule() now returns a ScheduledCancellable that no longer gives + access to the exception. Instead, any exception thrown out of a + scheduled Runnable is logged as a warning. + + This is a continuation of #28667, #36137 and also fixes #37708. + +commit 7f738e85412fa352bfb7fe7d2e17d18f79c4a48c +Author: David Turner +Date: Thu Jan 31 16:41:04 2019 +0000 + + Minor logging improvements (#38084) + + Fixes some log messages that caused some minor confusion when digging through a + log generated by a failing test. + +commit e066a59c899c94a00e2c6a9800f3903fd51273fa +Author: Jack Conradson +Date: Thu Jan 31 08:32:38 2019 -0800 + + Fix Painless void return bug (#38046) + + Painless now allows void functions and contexts to with a void return type to use + a return statement without a following expression. + +commit b8575c6aa30a335ed12bc5b38e4c79583bcd0755 +Author: Tim Brooks +Date: Thu Jan 31 09:31:22 2019 -0700 + + Update PutFollowAction serialization post-backport (#37989) + + This commit modifies the `PutFollowRequest` to reflect the fact that + active shard functionality has been backported to 6.7. + +commit 9923f0fe6a7002e43f47b3365a5ac6d413163405 +Author: Tal Levy +Date: Thu Jan 31 08:28:20 2019 -0800 + + fix a few versionAdded values in ElasticsearchExceptions (#37877) + + TooManyBucketsException was introduced in v6.2 + and SnapshotInProgressException was introduced in v6.7 + +commit 7a597cad0d8c05820eba26b60c70c3f576254cd5 +Author: Tanguy Leroux +Date: Thu Jan 31 16:53:28 2019 +0100 + + Reenable BWC tests after backport of #37899 (#38093) + + This commit adapts the version used in StartedShardEntry serialization + after the backport of #37899 and reenables bwc tests. + + Related to #37899 + Related to #38074 + +commit b7de8e1d1eeb1e8ab06b0817eea25b9f819e745f +Author: Alpar Torok +Date: Thu Jan 31 17:01:16 2019 +0200 + + Mute failing test + + Tracking #38100 + +commit f15d7b9b910907c56c1be04b86a1592c0adcd508 +Author: Alpar Torok +Date: Thu Jan 31 16:55:29 2019 +0200 + + Mute failing test + + Tracking #38027 + +commit ff7620fd7f59f6d40d772f7d5a85c4071d33c197 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Thu Jan 31 15:40:06 2019 +0100 + + Fail start on obsolete indices documentation (#37786) + + Added breaking changes documentation for node start up obsolete indices + detection. + + Issue #27073 + +commit 4710a7472f728c9e9bcebd1c25001fddbe3866ec +Author: Marios Trivyzas +Date: Thu Jan 31 16:33:05 2019 +0200 + + SQL: Implement FIRST/LAST aggregate functions (#37936) + + FIRST and LAST can be used with one argument and work similarly to MIN + and MAX but they are implemented using a Top Hits aggregation and + therefore can also operate on keyword fields. When a second argument is + provided then they return the first/last value of the first arg when its + values are ordered ascending/descending (respectively) by the values of + the second argument. Currently because of the usage of a Top Hits + aggregation FIRST and LAST cannot be used in the HAVING clause of a + GROUP BY query to filter on the results of the aggregation. + + Closes: #35639 + +commit 7487be3d3c9d82960e5a71743f4f3401bf9cfc74 +Author: Henning Andersen +Date: Thu Jan 31 15:31:01 2019 +0100 + + Un-mute NoMasterNodeIT.testNoMasterActionsWriteMasterBlock + +commit 91d7050a5b8d54413c2de55021cb049692d6f69f +Author: Boaz Leskes +Date: Thu Jan 31 15:26:09 2019 +0100 + + remove unused parser fields in RemoteResponseParsers + +commit a9b12b38f0a037cc588fefac23f5bc5e04745a52 +Author: Jason Tedor +Date: Thu Jan 31 09:19:49 2019 -0500 + + Push primary term to replication tracker (#38044) + + This commit pushes the primary term into the replication tracker. This + is a precursor to using the primary term to resolving ordering problems + for retention leases. Namely, it can be that out-of-order retention + lease sync requests arrive on a replica. To resolve this, we need a + tuple of (primary term, version). For this to be, the primary term needs + to be accessible in the replication tracker. As the primary term is part + of the replication group anyway, this change conceptually makes sense. + +commit 622fb7883b4e7c6de6dce2636108e334cca28cd3 +Author: Luca Cavanna +Date: Thu Jan 31 15:12:14 2019 +0100 + + Introduce ability to minimize round-trips in CCS (#37828) + + With #37566 we have introduced the ability to merge multiple search responses into one. That makes it possible to expose a new way of executing cross-cluster search requests, that makes CCS much faster whenever there is network latency between the CCS coordinating node and the remote clusters. The coordinating node can now send a single search request to each remote cluster, which gets reduced by each one of them. from + size results are requested to each cluster, and the reduce phase in each cluster is non final (meaning that buckets are not pruned and pipeline aggs are not executed). The CCS coordinating node performs an additional, final reduction, which produces one search response out of the multiple responses received from the different clusters. + + This new execution path will be activated by default for any CCS request unless a scroll is provided or inner hits are requested as part of field collapsing. The search API accepts now a new parameter called ccs_minimize_roundtrips that allows to opt-out of the default behaviour. + + Relates to #32125 + +commit ae9f4df361fd4e8848c47593f0e359db860c76c6 +Author: Armin Braun +Date: Thu Jan 31 14:35:11 2019 +0100 + + Don't Assert Ack on when Publish Timeout is 0 in Test (#38077) + + * Publish timeout is set to `0` so out of order processing of states on the node can lead to a `false` ack response + * See #30672 + * Closes #36813 + +commit 9f026bb8ad266a73da015a06fcf45d43417f0537 +Author: Alexander Reelsen +Date: Thu Jan 31 14:18:28 2019 +0100 + + Reduce object creation in Rounding class (#38061) + + This reduces objects creations in the rounding class (used by aggs) by properly + creating the objects only once. Furthermore a few unneeded ZonedDateTime objects + were created in order to create other objects out of them. This was + changed as well. + + Running the benchmarks shows a much faster performance for all of the + java time based Rounding classes. + +commit a536fa7755dcb3716308f2b6e35662013db3a21e +Author: Adrien Grand +Date: Thu Jan 31 13:57:42 2019 +0100 + + Treat put-mapping calls with `_doc` as a top-level key as typed calls. (#38032) + + Currently the put-mapping API assumes that because the type name is `_doc` then + it is dealing with a typeless put-mapping call. Yet we still allow running the + put-mapping API in a typed fashion with `_doc` as a type name. The current logic + triggers surprising errors when doing a typed put-mapping call with `_doc` as a + type name on an index that has a type already. + + This is a bit of a corner-case, but is more important on 6.x due to the fact + that using the index API with `_doc` as a type name triggers typed calls to the + put-mapping API with `_doc` as a type name. + +commit 3c439d3b92afa1b32b8123ebefbe0755a6280f4e +Author: Adrien Grand +Date: Thu Jan 31 13:52:09 2019 +0100 + + Fix test bug when testing the merging of mappings and templates. (#38021) + + This test performs a typed index call when it actually means to run a typeless + index call. + +commit 0154052335b19dcede19f5ac274252c7073e0f77 +Author: Josh Soref +Date: Thu Jan 31 07:09:36 2019 -0500 + + spelling: java script -- not JavaScript (#37057) + +commit cde126dbff1e443bdaa13b5d1b541dfb037db028 +Author: Tim Vernum +Date: Thu Jan 31 20:59:50 2019 +1100 + + Enable SSL in reindex with security QA tests (#37600) + + Update the x-pack/qa/reindex-tests-with-security integration tests to + run with TLS enabled on the Rest interface. + + Relates: #37527 + +commit 62b1874b927c15c239d7330baa841440373d6e62 +Author: Tanguy Leroux +Date: Thu Jan 31 10:36:12 2019 +0100 + + Disable BWC tests during backport (#38074) + + This pull request disables BWC tests while backporting #37899 to 6.x. + +commit 22d32900784600f99b907ef3e81aca94d5c78f9c +Author: Andrei Stefan +Date: Thu Jan 31 10:52:49 2019 +0200 + + SQL: Added SSL configuration options tests (#37875) + + * Added SSL configuration options tests + Removed the allow.self.signed option from the documentation since we allow + by default self signed certificates as well. + + * Added more tests + +commit 8309e0ce77321a919b13255f128203246cf5945f +Author: Adrien Grand +Date: Thu Jan 31 09:52:25 2019 +0100 + + Minor fixes in the release notes script. (#37967) + + The `:beats` label is actually `:Beats` in Github. + `:Core/Build` is now `:Core/Infra/Build`. + +commit 3332e332c729c0918f70561d70345703aa79eede +Author: Adrien Grand +Date: Thu Jan 31 09:51:03 2019 +0100 + + Fix typo in docs. (#38018) + + This has been introduced in #37871. + +commit 3dd4c96e8ea39908095c0812ea634f0e822cbfe9 +Author: David Pilato +Date: Thu Jan 31 09:17:02 2019 +0100 + + Update Lucene repo for 7.0.0-alpha2 (#37985) + + Let's help the users by giving them the right version to use for 7.0.0-alpha2 + +commit eadcb5f0f8f8e27a5719037968bf43ce5c03ccb8 +Author: David Turner +Date: Thu Jan 31 08:00:11 2019 +0000 + + Fix size of rolling-upgrade bootstrap config (#38031) + + Zen2 nodes will bootstrap themselves once they believe there to be no remaining + Zen1 master-eligible nodes in the cluster, as long as minimum_master_nodes is + satisfied. + + Today the bootstrap configuration comprises just the ids of the known + master-eligible nodes, and this might be too small to be safe. For instance, if + there are 5 master-eligible nodes (so that minimum_master_nodes is 3) then the + bootstrap configuration could comprise just 3 nodes, of which 2 form a quorum, + and this does not intersect other quorums that might arise, leading to a + split-brain. + + This commit fixes this by expanding the bootstrap configuration so that its + quorums satisfy minimum_master_nodes, by adding some of the IDs of the other + master-eligible nodes in the last-published cluster state. + +commit e0d5de33da5703bbcffe4dd179947757499e2bc0 +Author: Tal Levy +Date: Wed Jan 30 23:57:56 2019 -0800 + + fix DateIndexNameProcessorTests offset pattern (#38069) + + `XX` was being used to represent an offset pattern, + it should be `ZZ` + + Fixes #38067. + +commit b94acb608b58e7918c2ec1296db83dbc17391fe3 +Author: Alexander Reelsen +Date: Thu Jan 31 08:55:40 2019 +0100 + + Speed up converting of temporal accessor to zoned date time (#37915) + + The existing implementation was slow due to exceptions being thrown if + an accessor did not have a time zone. This implementation queries for + having a timezone, local time and local date and also checks for an + instant preventing to throw an exception and thus speeding up the conversion. + + This removes the existing method and create a new one named + DateFormatters.from(TemporalAccessor accessor) to resemble the naming of + the java time ones. + + Before this change an epoch millis parser using the toZonedDateTime + method took approximately 50x longer. + + Relates #37826 + +commit 160d1bd4dd59fd669d062e7f016bf14ac6be1fe4 +Author: Alexander Reelsen +Date: Thu Jan 31 08:52:35 2019 +0100 + + Work around JDK8 timezone bug in tests (#37968) + + The timezone GMT0 cannot be properly parsed on java8. + The randomZone() method now excludes GMT0, if java8 is used. + + Closes #37814 + +commit 1a93976ff7385ea1a505aef658e938da310431b7 +Author: Nhat Nguyen +Date: Thu Jan 31 02:45:42 2019 -0500 + + Correct arg names when update mapping/settings from leader (#38063) + + These two arguments are not named incorrectly and caused confusion. + +commit a8596de31fff2372fadd3f48dd97c6d674f2eb58 +Author: Tim Vernum +Date: Thu Jan 31 18:06:05 2019 +1100 + + Introduce ssl settings to reindex from remote (#37527) + + Adds reindex.ssl.* settings for reindex from remote. + + This uses the ssl-config/ internal library to parse and load SSL + configuration and files. This is applied when using the low level + rest client to connect to a remote ES node + + Relates: #37287 + Resolves: #29755 + +commit f5398d65119e7ea851fca9e806dc7677a987d11e +Author: Nhat Nguyen +Date: Thu Jan 31 00:57:27 2019 -0500 + + Mute testRetentionLeasesSyncOnExpiration + + Tracked at #37963 + +commit 89bffc25de064dc20e5d1cd9d4d509bfb61d7612 +Author: Jason Tedor +Date: Wed Jan 30 20:37:52 2019 -0500 + + Mute failing date index name processor test + + This test is repeatedly failing, so this commit mutes it. + + Relates #38067 + +commit a6a534f1f0b19075dc26455c6f2bd2a692628324 +Author: Jason Tedor +Date: Wed Jan 30 20:34:27 2019 -0500 + + Reenable BWC testing after retention lease stats (#38062) + + This commit adjusts the BWC version on retention leases in stats, so + with this we also reenable BWC testing. + +commit b11732104fa5a611b5d47bc59efd594e9cbdab89 +Author: Boaz Leskes +Date: Wed Jan 30 20:14:59 2019 -0500 + + Move watcher to use seq# and primary term for concurrency control (#37977) + + * move watcher to seq# occ + + * top level set + + * fix parsing and missing setters + + * share toXContent for PutResponse and rest end point + + * fix redacted password + + * fix username reference + + * fix deactivate-watch.asciidoc have seq no references + + * add seq# + term to activate-watch.asciidoc + + * more doc fixes + +commit dad41c2b7f1c2bb0c2396036d550c5bd5fb381a5 +Author: Jake Landis +Date: Wed Jan 30 17:38:47 2019 -0600 + + ILM setPriority corrections for a 0 value (#38001) + + This commit fixes the test case that ensures only a priority + less then 0 is used with testNonPositivePriority. This also + allows the HLRC to support a value of 0. + + Closes #37652 + +commit 6935d3d5fa020055145dfff6b5a7cdeac5ac3d83 +Author: Jason Tedor +Date: Wed Jan 30 18:18:30 2019 -0500 + + Temporarily disable BWC for retention lease stats (#38049) + + This commit temporarily disables BWC testing while backporting a change + to expose retention leases in stats. + +commit 7c738fd2414a5a11954d201a4af49307472b99fc +Author: Tal Levy +Date: Wed Jan 30 15:09:17 2019 -0800 + + Skip Shrink when numberOfShards not changed (#37953) + + Previously, ShrinkAction would fail if + it was executed on an index that had + the same number of shards as the target + shrunken number. + + This PR introduced a new BranchingStep that + is used inside of ShrinkAction to branch which + step to move to next, depending on the + shard values. So no shrink will occur if the + shard count is unchanged. + +commit b88bdfe958ccedb96f624bbd796ab10c5e5f70cb +Author: Tim Brooks +Date: Wed Jan 30 15:40:49 2019 -0700 + + Add dispatching to `HandledTransportAction` (#38050) + + This commit allows implementors of the `HandledTransportAction` to + specify what thread the action should be executed on. The motivation for + this commit is that certain CCR requests should be performed on the + generic threadpool. + +commit 54dbf9469cb31b9b832f3e02243329a8dd0235b6 +Author: Jay Modi +Date: Wed Jan 30 14:24:29 2019 -0700 + + Update httpclient for JDK 11 TLS engine (#37994) + + The apache commons http client implementations recently released + versions that solve TLS compatibility issues with the new TLS engine + that supports TLSv1.3 with JDK 11. This change updates our code to + use these versions since JDK 11 is a supported JDK and we should + allow the use of TLSv1.3. + +commit aeab55e8d125efdfe66e4c9ad784c9d8c7fe6736 +Author: Tim Brooks +Date: Wed Jan 30 14:13:23 2019 -0700 + + Reduce flaxiness of ccr recovery timeouts test (#38035) + + This fixes #38027. Currently we assert that all shards have failed. + However, it is possible that some shards do not have segement files + created yet. The action that we block is fetching these segement files + so it is possible that some shards successfully recover. + + This commit changes the assertion to ensure that at least some of the + shards have failed. + +commit daafcb66256aa209673b3c5b7bc57af8ed51a3b5 +Author: Michael Basnight +Date: Wed Jan 30 14:32:17 2019 -0600 + + Fix ILM status to allow unknown fields (#38043) + + The ILM status parser did not allow for unknown fields. This commit + fixes that and adds an xContentTester to the response test. + + Relates #36938 + +commit 14c571532a1557424e02d05382c758233134e781 +Author: Michael Basnight +Date: Wed Jan 30 14:31:55 2019 -0600 + + Fix ILM Lifecycle Policy to allow unknown fields (#38041) + + A few of the ILM Lifecycle Policy and classes did not allow for unknown + fields. This commit sets those parsers and fixes the tests for the + parsers. + + Relates #36938 + +commit 945ad05d54c3dabc5ab0a39e6bac6dd287f2b00b +Author: Michael Basnight +Date: Wed Jan 30 14:31:16 2019 -0600 + + Update verify repository to allow unknown fields (#37619) + + The subparser in verify repository allows for unknown fields. This + commit sets the value to true for the parser and modifies the test such + that it accurately tests it. + + Relates #36938 + +commit be788160efd23fed3eba5ef1d431cb1cb3e88cf6 +Author: David Roberts +Date: Wed Jan 30 20:12:20 2019 +0000 + + [ML] Datafeed deprecation checks (#38026) + + Deprecation checks for the ML datafeed + query and aggregations. + +commit 81c443c9de81f3cd27b2050c0a85ff29d147cd73 +Author: David Turner +Date: Wed Jan 30 20:09:15 2019 +0000 + + Deprecate minimum_master_nodes (#37868) + + Today we pass `discovery.zen.minimum_master_nodes` to nodes started up in + tests, but for 7.x nodes this setting is not required as it has no effect. + This commit removes this setting so that nodes are started with more realistic + configurations, and deprecates it. + +commit 6a78b6a31cfe0f06f2656672db12f1d5edfb5aaf +Author: Jake Landis +Date: Wed Jan 30 13:12:13 2019 -0600 + + Remove types from watcher docs (#38002) + + Types have been deprecated and this commit removes the documentation + for specifying types in the index action, and search input/transform. + + Relates #37594 #35190 + +commit 36ee78d92468901be372b5218890f4c3e59f564d +Author: Jack Conradson +Date: Wed Jan 30 11:01:45 2019 -0800 + + Add test coverage for Painless general casting of boolean and Boolean (#37780) + + This adds test coverage for general casts in Painless between boolean and other types and Boolean and other types. + +commit 5433af28e33daeda6d30e8cd41ac1e69a548334c +Author: Martijn van Groningen +Date: Wed Jan 30 19:33:00 2019 +0100 + + Fixed test bug, lastFollowTime is null if there are no follower indices. + +commit cac6b8e06f051d68919faf6081f1c87fa5b6757d +Author: Lee Hinman +Date: Wed Jan 30 11:24:18 2019 -0700 + + Add ECS schema for user-agent ingest processor (#37727) (#37984) + + * Add ECS schema for user-agent ingest processor (#37727) + + This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs). + So rather than something like this: + + ``` + { + "patch" : "3538", + "major" : "70", + "minor" : "0", + "os" : "Mac OS X 10.14.1", + "os_minor" : "14", + "os_major" : "10", + "name" : "Chrome", + "os_name" : "Mac OS X", + "device" : "Other" + } + ``` + + The structure is now like this: + + ``` + { + "name" : "Chrome", + "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "os" : { + "name" : "Mac OS X", + "version" : "10.14.1", + "full" : "Mac OS X 10.14.1" + }, + "device" : "Other", + "version" : "70.0.3538.102" + } + ``` + + This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not + supported. + + Resolves #37329 + + * Remove `ecs` setting from docs + +commit a070b8acc004740a001bd3b01e6fc6417a9e7919 +Author: Armin Braun +Date: Wed Jan 30 19:21:09 2019 +0100 + + Extract TransportRequestDeduplication from ShardStateAction (#37870) + + * Extracted the logic for master request duplication so it can be reused by the snapshotting logic + * Removed custom listener used by `ShardStateAction` to not leak these into future users of this class + * Changed semantics slightly to get rid of redundant instantiations of the composite listener + * Relates #37686 + +commit 6500b0cbd78a0e3968ecb54a33b59a2325059ff9 +Author: Jason Tedor +Date: Wed Jan 30 13:20:40 2019 -0500 + + Expose retention leases in shard stats (#37991) + + This commit exposes retention leases via shard-level stats. + +commit c468b2f7caff023d285e8a01e39a3873973eaf28 +Author: Jason Tedor +Date: Wed Jan 30 12:56:58 2019 -0500 + + Make primary terms fields private in index shard (#38036) + + This commit encapsulates the primary terms fields in index shard. This + is a precursor to pushing the operation primary term down to the + replication tracker. + +commit 9782aaa1b85b37933d08daa41388a5f47dcf3309 +Author: Benjamin Trent +Date: Wed Jan 30 11:56:24 2019 -0600 + + ML: Add reason field in JobTaskState (#38029) + + * ML: adding reason to job failure status + + * marking reason as nullable + + * Update AutodetectProcessManager.java + +commit ed460c28159172b4ea06800dc909ed72a5ace800 +Author: Nhat Nguyen +Date: Wed Jan 30 12:28:24 2019 -0500 + + Log flush_stats and commit_stats in testMaybeFlush + + This test failed a few times over the last several months. It seems that + we triggered a flush, but CI was too slow to finish it in several + seconds. I added the flush stats and commit stats and unmuted this test. + We should have a good clue if this test fails again. + + Relates #37896 + +commit 2cbc6888a2dac2eb577808a379485fd8a5118bd6 +Author: Michael Basnight +Date: Wed Jan 30 11:31:59 2019 -0600 + + HLRC: Fix strict setting exception handling (#37247) + + The LLRC's exception handling for strict mode was previously throwing an + exception the HLRC assumed was an error response. This is not the case + if the result is valid in strict mode, as it will return the proper + response wrapped in an exception with warnings. This commit fixes the + HLRC such that it no longer spews if it encounters a strict LLRC + response. + + Closes #37090 + +commit e97718245d206e260a4b696565be2eb0cc0801d3 +Author: Nik Everett +Date: Wed Jan 30 11:48:34 2019 -0500 + + Test: Enable strict deprecation on all tests (#36558) + + This drops the option for tests to disable strict deprecation mode in + the low level rest client in favor of configuring expected warnings on + any calls that should expect warnings. This behavior is + paranoid-by-default which is generally the right way to handle + deprecations and tests in general. + +commit 21e392e95e6ab5b293a46f7a2af84c0fa4778029 +Author: Colin Goodheart-Smithe +Date: Wed Jan 30 16:32:58 2019 +0000 + + Removes typed calls from YAML REST tests (#37611) + + This PR attempts to remove all typed calls from our YAML REST tests. The PR adds include_type_name: false to create index requests that use a mapping and also to put mapping requests. It also removes _type from index requests where they haven't already been removed. The PR ignores tests named *_with_types.yml since this are specifically testing typed API behaviour. + + The change also includes changing the test harness to add the type _doc to index, update, get and bulk requests that do not specify the document type when the test is running against a mixed 7.x/6.x cluster. + +commit 2f7776c8b78c40a538827a195014d52c1d1e5ccf +Author: David Roberts +Date: Wed Jan 30 16:26:28 2019 +0000 + + Switch default time format for ingest from Joda to Java for v7 (#37934) + + Date formats with and without the "8" prefix are now all treated + as Java time formats, so that ingest does the same as mappings + in this respect. + +commit ecbaa388648da7327bf00611a4640ac3475326c1 +Author: Christoph Büscher +Date: Wed Jan 30 17:17:54 2019 +0100 + + Remove deprecated Plugin#onModule extension points (#37866) + + Removes some guice index level extension point marked as @Deprecated since at + least 6.0. They served as a signpost for plugin authors upgrading from 2.x but + this shouldn't be relevant in 7.0 anymore. + +commit 23805fa41ad40e02611a765f96b2188a1e730e64 +Author: Igor Motov +Date: Wed Jan 30 09:20:30 2019 -0500 + + Geo: Fix Empty Geometry Collection Handling (#37978) + + Fixes handling empty geometry collection and re-enables + testParseGeometryCollection test. + + Fixes #37894 + +commit 53e80e98148a714d90da8f49bb283d290d737109 +Author: Albert Zaharovits +Date: Wed Jan 30 16:11:44 2019 +0200 + + Fix failure in test code ClusterPrivilegeTests + + Closes #38030 + +commit 8280a2066444391979a7a35a73c39a0d19c3186e +Author: Benjamin Trent +Date: Wed Jan 30 06:51:11 2019 -0600 + + ML: Add upgrade mode docs, hlrc, and fix bug (#37942) + + * ML: Add upgrade mode docs, hlrc, and fix bug + + * [DOCS] Fixes build error and edits text + + * adjusting docs + + * Update docs/reference/ml/apis/set-upgrade-mode.asciidoc + + Co-Authored-By: benwtrent + + * Update set-upgrade-mode.asciidoc + + * Update set-upgrade-mode.asciidoc + +commit f3379940c6ceb74a3bcb553c90151e1650a2718d +Author: Alpar Torok +Date: Wed Jan 30 14:41:35 2019 +0200 + + Fix the packer cache script (#38023) + + The script is used to create a cache on ephemeral CI workers. + Changes: + - create and use a `pullFixture` task that always exists regardless + of docker support + - wire dependencies correctly so any pre fixture setup runs for pull + as well + - set up java env vars so bwc versions can build + +commit b91d587275cbce37142d46d0f2ef95cdc94ae87e +Author: Luca Cavanna +Date: Wed Jan 30 12:05:54 2019 +0100 + + Move SearchHit and SearchHits to Writeable (#37931) + + This allowed to make SearchHits immutable, while quite a few fields in + SearchHit have to stay mutable unfortunately. + + Relates to #34389 + +commit ba285a56a72655d189ee11aeed8c9eb8d2043ff9 +Author: Jason Tedor +Date: Wed Jan 30 05:25:17 2019 -0500 + + Fix limit on retaining sequence number (#37992) + + We only assign non-negative sequence numbers to operations, so the lower + limit on retaining sequence numbers should be that it is non-negative + only. + +commit 3865435a01a5b60465f37f036953c306e950a75b +Author: Martijn van Groningen +Date: Wed Jan 30 11:02:16 2019 +0100 + + Docs test fix, wait for shards active. + + (a restore needs to be complete, which happens in the background and + by default the ccr put follow api doesn't wait for this) + + (this was a recent change and the pr that added this docs test, + did not include this change) + + Relates to #37917 + +commit e959dbaa998482bee009f3064b6e3edea60301e5 +Author: Martijn van Groningen +Date: Wed Jan 30 10:49:42 2019 +0100 + + Revert "Revert "Documented default values for index follow request parameters. (#37917)"" + + This reverts commit 7205833f92df93bfdb09296aafade865c57af0d4. + +commit 7205833f92df93bfdb09296aafade865c57af0d4 +Author: Adrien Grand +Date: Wed Jan 30 10:43:57 2019 +0100 + + Revert "Documented default values for index follow request parameters. (#37917)" + + This reverts commit 4da7a446489674beff665a3093037fdb25f93787. + +commit 9ec4abc31eeaf7f60a4ff3597addcb154b07a42e +Author: Alexander Reelsen +Date: Wed Jan 30 10:47:12 2019 +0100 + + Ensure date parsing BWC compatibility (#37929) + + In order to retain BWC this changes the java date formatters to be able to + parse nanoseconds resolution, even if only milliseconds are supported. + This used to work on joda time as well so that a user could store a date + like `2018-10-03T14:42:44.613469+0000` and then just loose the precision + on anything lower than millisecond level. + +commit 908c8def063e529ee4f1aa146d43132b9dafd66e +Author: Andrei Stefan +Date: Wed Jan 30 11:34:47 2019 +0200 + + SQL: Skip the nested and object field types in case of an ODBC request (#37948) + +commit c8af0f4bfafb1ec15a59b91b0e04f157a503d615 +Author: Adrien Grand +Date: Wed Jan 30 10:31:51 2019 +0100 + + Use mappings to format doc-value fields by default. (#30831) + + Doc-value fields now return a value that is based on the mappings rather than + the script implementation by default. + + This deprecates the special `use_field_mapping` docvalue format which was added + in #29639 only to ease the transition to 7.x and it is not necessary anymore in + 7.0. + +commit b63b50b945d4c44c70d56a9ce2ba9a25dd897147 +Author: Adrien Grand +Date: Wed Jan 30 10:28:24 2019 +0100 + + Give precedence to index creation when mixing typed templates with typeless index creation and vice-versa. (#37871) + + Currently if you mix typed templates and typeless index creation or typeless + templates and typed index creation then you will end up with an error because + Elasticsearch tries to create an index that has multiple types: `_doc` and + the explicit type name that you used. + + This commit proposes to give precedence to the index creation call so that + the type from the template will be ignored if the index creation call is + typeless while the template is typed, and the type from the index creation + call will be used if there is a typeless template. + + This is consistent with the fact that index creation already "wins" if a field + is defined differently in the index creation call and in a template: the + definition from the index creation call is used in such cases. + + Closes #37773 + +commit 4dee3f741846ee8e7ccc59c09c550e0c88f10039 +Author: Tim Vernum +Date: Wed Jan 30 20:13:48 2019 +1100 + + Add classifier to tar.gz in docker compose (#38011) + + The distribution now includes a platform specific classifier that the + docker build wasn't taking into account. + + Relates: #37881 + +commit 4da7a446489674beff665a3093037fdb25f93787 +Author: Martijn van Groningen +Date: Wed Jan 30 09:37:41 2019 +0100 + + Documented default values for index follow request parameters. (#37917) + +commit 2732bb5cf3d854ab404a5ebe44a2f116cc3f1073 +Author: Jim Ferenczi +Date: Wed Jan 30 08:46:14 2019 +0100 + + Fix fetch source option in expand search phase (#37908) + + This change fixes the copy of the fetch source option into the + expand search request that is used to retrieve the documents of each + collapsed group. + + Closes #23829 + +commit 5dcc805dc9fad2fde98610145639b5faa128ec11 +Author: Jim Ferenczi +Date: Wed Jan 30 08:45:50 2019 +0100 + + Restore a noop _all metadata field for 6x indices (#37808) + + This commit restores a noop version of the AllFieldMapper that is instanciated only + for indices created in 6x. We need this metadata field mapper to be present in this version + in order to allow the upgrade of indices that explicitly disable _all (enabled: false). + The mapping of these indices contains a reference to the _all field that we cannot remove + in 7 so we'll need to keep this metadata mapper in 7x. Since indices created in 6x will not + be compatible with 8, we'll remove this noop mapper in the next major version. + + Closes #37429 + +commit f51bc00fcf18fa0073641df7cf1caff7e68a9104 +Author: Martijn van Groningen +Date: Wed Jan 30 07:58:26 2019 +0100 + + Added ccr to xpack usage infrastructure (#37256) + + * Added ccr to xpack usage infrastructure + + Closes #37221 + +commit 99129d7786229ab2f537784ba148740a3053eb70 +Author: Tim Vernum +Date: Wed Jan 30 17:51:11 2019 +1100 + + Fix exit code for Security CLI tools (#37956) + + The certgen, certutil and saml-metadata tools did not correctly return + their exit code to the calling shell. + + These commands now explicitly exit with the code that was returned + from the main(args, terminal) method. + +commit 57823c484fddafb65accce261621a3b051e647db +Author: Armin Braun +Date: Wed Jan 30 06:22:31 2019 +0100 + + Streamline S3 Repository- and Client-Settings (#37393) + + * Make repository settings override static settings + * Cache clients according to settings + * Introduce custom implementations for the AWS credentials here to be able to use them as part of a hash key + +commit f5b9b4d89c9e28dba020cd53c39b582fb01d95bd +Author: Marios Trivyzas +Date: Wed Jan 30 15:32:45 2019 +1100 + + Add version 6.6.1 (#37975) + +commit 55b916afc04e7867d6fc5070445c3551b8d5a364 +Author: Tim Brooks +Date: Tue Jan 29 15:58:31 2019 -0700 + + Ensure task metadata not null in follow test (#37993) + + This commit fixes a potential race in the IndexFollowingIT. Currently it + is possible that we fetch the task metadata, it is null, and that throws + a null pointer exception. Assertbusy does not catch null pointer + exceptions. This commit assertions that the metadata is not null. + +commit 0470ee1fcc757296e3f7f1f168a3a98539c01f44 +Author: markharwood +Date: Tue Jan 29 21:24:08 2019 +0000 + + Docs fix - missing callout + +commit b889221f75048bebbc7084bf3b3f341eac30e34c +Author: markharwood +Date: Tue Jan 29 20:52:41 2019 +0000 + + Types removal - deprecate include_type_name with index templates (#37484) + + Added deprecation warnings for use of include_type_name in put/get index templates. + HLRC changes: + GetIndexTemplateRequest has a new client-side class which is a copy of server's GetIndexTemplateResponse but modified to be typeless. + PutIndexTemplateRequest has a new client-side counterpart which doesn't use types in the mappings + Relates to #35190 + +commit 193017672a3a283a2e5bb85f1002f656a4e5e156 +Author: jimczi +Date: Tue Jan 29 20:31:24 2019 +0100 + + Handle completion suggestion without contexts + + This change fixes the handling of completion suggestion without contexts. + + Relates #36996 + +commit f3f9cabd67681009f5eadee369b1dc1e24d8c67d +Author: Tim Brooks +Date: Tue Jan 29 12:29:06 2019 -0700 + + Add timeout for ccr recovery action (#37840) + + This is related to #35975. It adds a action timeout setting that allows + timeouts to be applied to the individual transport actions that are + used during a ccr recovery. + +commit e9332331a39efdb42be204c7df0f04d4088d0d6e +Author: Marios Trivyzas +Date: Tue Jan 29 21:20:09 2019 +0200 + + SQL: Make error msg for validation of 2nd arg of PERCENTILE[_RANK] consistent (#37937) + + Use `first` and `second` instead of `1st` and `2nd`. + +commit 8e5f9c4b142f6ed0b5b0424c7de5d6c309989164 +Author: Ryan Ernst +Date: Tue Jan 29 11:18:30 2019 -0800 + + Add OS/architecture classifier to distributions (#37881) + + This commit adds classifiers to the distributions indicating the + OS (for archives) and platform. The current OSes are for windows, darwin (ie + macos) and linux. This change will allow future OS/architecture specific + changes to the distributions. Note the docs using distribution links + have been updated, but will be reworked in a followup to make OS + specific instructions for the archives. + +commit 697b2fbe520942a2212e1778b5b251138db93425 +Author: Albert Zaharovits +Date: Tue Jan 29 21:10:03 2019 +0200 + + Remove implicit index monitor privilege (#37774) + + Restricted indices (currently only .security-6 and .security) are special + internal indices that require setting the `allow_restricted_indices` flag + on every index permission that covers them. If this flag is `false` + (default) the permission will not cover these and actions against them + will not be authorized. + However, the monitoring APIs were the only exception to this rule. + + This exception is herein forfeited and index monitoring privileges have to be + granted explicitly, using the `allow_restricted_indices` flag on the permission, + as is the case for any other index privilege. + +commit 34d61d3231261794a58a918fce9642cf6699fbab +Author: Benjamin Trent +Date: Tue Jan 29 12:51:34 2019 -0600 + + ML: ignore unknown fields for JobTaskState (#37982) + +commit 9ca26b7e63b4f75f82842c432377ccee5cdc1e6f +Author: Julie Tibshirani +Date: Tue Jan 29 10:51:07 2019 -0800 + + Remove more references to type in docs. (#37946) + + * Update the top-level 'getting started' guide. + * Remove custom types from the painless getting started documentation. + * Fix an incorrect references to '_doc' in the cardinality query docs. + * Update the _update docs to use the typeless API format. + +commit 00ace369af0c9b85dca78a57b083c65b7ffd823c +Author: Tim Brooks +Date: Tue Jan 29 11:47:29 2019 -0700 + + Use `CcrRepository` to init follower index (#35719) + + This commit modifies the put follow index action to use a + CcrRepository when creating a follower index. It routes + the logic through the snapshot/restore process. A + wait_for_active_shards parameter can be used to configure + how long to wait before returning the response. + +commit d05a4b9d14bbcb0f8900d871478d3c97b72c5ba1 +Author: Albert Zaharovits +Date: Tue Jan 29 18:56:20 2019 +0200 + + Get Aliases with wildcard exclusion expression (#34230) + + This commit adds the code in the HTTP layer that will parse exclusion wildcard + expressions. + The existing code issues 404s for wildcards as well as explicit indices. + But, in general, in an expression with exclude wildcards (-...*) following other + include wildcards, there is no way to tell if the include wildcard produced no + results or they were subsequently excluded. + Therefore, the proposed change is breaking the behavior of 404s for + wildcards. Specifically, no 404s will be returned for wildcards, even + if they are not followed by exclude wildcards or the exclude wildcards + could not possibly exclude what has previously been included. + Only explicitly requested aliases will be called out as missing. + +commit 1579ac032b690d98817f38be6911e758256bd08f +Author: markharwood +Date: Tue Jan 29 16:43:24 2019 +0000 + + Added missing eclipse-build.gradle files (#37980) + + Eclipse build files were missing so .eclipse project files were not being generated. + + Closes #37973 + +commit 5f106a27eaaf6deded97b421230065aa3b487409 +Author: David Roberts +Date: Tue Jan 29 15:41:35 2019 +0000 + + [ML] Add meta information to all ML indices (#37964) + + This change adds a _meta field storing the version in which + the index mappings were last updated to the 3 ML indices + that didn't previously have one: + + - .ml-annotations + - .ml-meta + - .ml-notifications + + All other ML indices already had such a _meta field. + + This field will be useful if we ever need to automatically + update the index mappings during a future upgrade. + +commit 218df3009a8d3224c416f9ab05893e637714e366 +Author: Boaz Leskes +Date: Tue Jan 29 10:23:05 2019 -0500 + + Move update and delete by query to use seq# for optimistic concurrency control (#37857) + + The delete and update by query APIs both offer protection against overriding concurrent user changes to the documents they touch. They currently are using internal versioning. This PR changes that to rely on sequences numbers and primary terms. + + Relates #37639 + Relates #36148 + Relates #10708 + +commit 6d1693ff49974b1aef9741551df33ad3e851cec4 +Author: David Kyle +Date: Tue Jan 29 15:09:40 2019 +0000 + + [ML] Prevent submit after autodetect worker is stopped (#37700) + + Runnables can be submitted to + AutodetectProcessManager.AutodetectWorkerExecutorService + without error after it has been shutdown which can lead + to requests timing out as their handlers are never called + by the terminated executor. + + This change throws an EsRejectedExecutionException if a + runnable is submitted after after the shutdown and calls + AbstractRunnable.onRejection on any tasks not run. + + Closes #37108 + +commit 3c9f7031b92dab442305eb9bbc1df48e5d4aa6da +Author: Yannick Welsch +Date: Tue Jan 29 15:41:05 2019 +0100 + + Enforce cluster UUIDs (#37775) + + This commit adds join validation around cluster UUIDs, preventing a node to join a cluster if it was + previously part of another cluster. The commit introduces a new flag to the cluster state, + clusterUUIDCommitted, which denotes whether the node has locked into a cluster with the given + uuid. When a cluster is committed, this flag will turn to true, and subsequent cluster state updates + will keep the information about committal. Note that coordinating-only nodes are still free to switch + clusters at will (after restart), as they don't carry any persistent state. + +commit 09a11a34effe7e9f81157560f5936b2790c50201 +Author: Luca Cavanna +Date: Tue Jan 29 15:31:49 2019 +0100 + + Remove clusterAlias instance member from QueryShardContext (#37923) + + The clusterAlias member is only used in the copy constructor, to be able + to reconstruct the fully qualified index. It is also possible to remove + the instance member and add a private constructor that accepts the already built Index object which contains the cluster alias. + +commit 65a9b61a9162d4a9f5181b164ce4d23ee343b56e +Author: Boaz Leskes +Date: Tue Jan 29 09:18:05 2019 -0500 + + Add Seq# based optimistic concurrency control to UpdateRequest (#37872) + + The update request has a lesser known support for a one off update of a known document version. This PR adds an a seq# based alternative to power these operations. + + Relates #36148 + Relates #10708 + +commit 5d1964bcbf5c4dcdf9ea8e567a1b4211f47aff33 +Author: Tanguy Leroux +Date: Tue Jan 29 15:09:40 2019 +0100 + + Ignore shard started requests when primary term does not match (#37899) + + This commit changes the StartedShardEntry so that it also contains the + primary term of the shard to start. This way the master node can also + checks that the primary term from the start request is equal to the current + shard's primary term in the cluster state, and it can ignore any shard + started request that would concerns a previous instance of the shard that + would have been allocated to the same node. + + Such situation are likely to happen with frozen (or restored) indices and + the replication of closed indices, because with replicated closed indices + the shards will be initialized again after the index is closed and can + potentially be re initialized again if the index is reopened as a frozen + index. In such cases the lifecycle of the shards would be something like: + * shard is STARTED + * index is closed + * shards is INITIALIZING (index state is CLOSED, primary term is X) + * index is reopened + * shards are INITIALIZING again (index state is OPENED, potentially frozen, + primary term is X+1) + + Adding the primary term to the shard started request will allow to discard + potential StartedShardEntry requests received by the master node if the + request concerns the shard with primary term X because it has been + moved/reinitialized in the meanwhile under the primary term X+1. + + Relates to #33888 + +commit 7f1784e9f9a3fe370a398875c07f03cd34133671 +Author: Armin Braun +Date: Tue Jan 29 15:08:11 2019 +0100 + + Remove Dead MockTransport Code (#34044) + + * All these methods are unused + +commit 2325fb9cb3530607475233e7c7281e5ff7241001 +Author: Luca Cavanna +Date: Tue Jan 29 14:58:11 2019 +0100 + + Remove test only SearchShardTarget constructor (#37912) + + Remove SearchShardTarget test only constructor and replace all the usages with calls to the other constructor that accepts a ShardId. + +commit 42eec55837f99167b36ebc2dbffb3d6bd52d7279 +Author: Luca Cavanna +Date: Tue Jan 29 14:57:33 2019 +0100 + + Replace failure.get().addSuppressed with failure.accumulateAndGet() (#37649) + + Also add a test for concurrent incoming failures + +commit a6d4838a6779b4073008dc1f2e75a3ffb7cdfe0f +Author: Luca Cavanna +Date: Tue Jan 29 14:56:22 2019 +0100 + + Clean up allowPartialSearchResults serialization (#37911) + + When serializing allowPartialSearchResults to the shards through ShardSearchTransportRequest, we use an optional boolean field, though + the corresponding instance member is declared `boolean` which can never + be null. We also have an assert to verify that the incoming search + request provides a non-null value for the flag, and a comment explaining + that null should be considered a bug. + + This commit makes the allowPartialSearchResults method in + ShardSearchRequest return a `boolean` rather than a `Boolean` and + changes the serialization from optional to non optional, in a bw comp manner. + +commit 4f4113e96474efde4ebeee351ff947ae22f6723d +Author: Przemyslaw Gomulka +Date: Tue Jan 29 14:53:55 2019 +0100 + + Rename security audit.log to _audit.json (#37916) + + in order to keep json logs consistent the security audit logs are renamed from .log to .json + relates #32850 + +commit 460f10ce602f067816c2615f035a3d42f42d3ed1 +Author: Tanguy Leroux +Date: Tue Jan 29 13:15:58 2019 +0100 + + Close Index API should force a flush if a sync is needed (#37961) + + This commit changes the TransportVerifyShardBeforeCloseAction so that it issues a + forced flush, forcing the translog and the Lucene commit to contain the same max seq + number and global checkpoint in the case the Translog contains operations that were + not written in the IndexWriter (like a Delete that touches a non existing doc). This way + the assertion added in #37426 won't trip. + + Related to #33888 + +commit 504a89feaf7c7eed55e7c44e7606fbc9ca7dae9a +Author: Yannick Welsch +Date: Tue Jan 29 12:43:04 2019 +0100 + + Step down as master when configured out of voting configuration (#37802) + + Abdicates to another master-eligible node once the active master is reconfigured out of the voting + configuration, for example through the use of voting configuration exclusions. + + Follow-up to #37712 + +commit 827c4f656711a4a8897a5139be386119a33b5f69 +Author: Yannick Welsch +Date: Tue Jan 29 10:44:01 2019 +0100 + + Make Version.java aware of 6.x Lucene upgrade + + Relates to #37913 + +commit eceb3185c79b9a777e6f53e899bc61b4f10ffd42 +Author: Henrique Gonçalves <36671768+up201608320@users.noreply.github.com> +Date: Tue Jan 29 09:06:50 2019 +0000 + + [ML] Make GetJobStats work with arbitrary wildcards and groups (#36683) + + The /_ml/anomaly_detectors/{job}/_stats endpoint now + works correctly when {job} is a wildcard or job group. + + Closes #34745 + +commit ebe9c9523080133d547f04d9d67c78c8d6f95fbd +Author: Dimitris Athanasiou +Date: Tue Jan 29 10:23:50 2019 +0200 + + [ML] Audit all errors during job deletion (#37933) + + This commit moves the auditing of job deletion related errors + to the final listener in the job delete action. This ensures + any error that occurs during job deletion is audited. + +commit 891320f5ac5d92a89dbe90f050919d0bd9f671a0 +Author: Przemyslaw Gomulka +Date: Tue Jan 29 07:20:09 2019 +0100 + + Elasticsearch support to JSON logging (#36833) + + In order to support JSON log format, a custom pattern layout was used and its configuration is enclosed in ESJsonLayout. Users are free to use their own patterns, but if smooth Beats integration is needed, they should use ESJsonLayout. EvilLoggerTests are left intact to make sure user's custom log patterns work fine. + + To populate additional fields node.id and cluster.uuid which are not available at start time, + a cluster state update will have to be received and the values passed to log4j pattern converter. + A ClusterStateObserver.Listener is used to receive only one ClusteStateUpdate. Once update is received the nodeId and clusterUUid are set in a static field in a NodeAndClusterIdConverter. + + Following fields are expected in JSON log lines: type, tiemstamp, level, component, cluster.name, node.name, node.id, cluster.uuid, message, stacktrace + see ESJsonLayout.java for more details and field descriptions + + Docker log4j2 configuration is now almost the same as the one use for ES binary. + The only difference is that docker is using console appenders, whereas ES is using file appenders. + + relates: #32850 + +commit 6ed35fbb945b35a65b0c56dbd3143f006c69e447 +Author: Like +Date: Tue Jan 29 08:01:09 2019 +0800 + + Support merge nested Map in list for JIRA configurations (#37634) + + This commit allows JIRA API fields that require a list of key/value + pairs (maps), such as JIRA "components" to use use template snippets + (e.g. {{ctx.payload.foo}}). Prior to this change the templated value + (not the de-referenced value) would be sent via the API and error. + + Closes #30068 + +commit 09b6028e153948e90c9f19d64a1de6aee5d68464 +Author: Ryan Ernst +Date: Mon Jan 28 15:57:27 2019 -0800 + + Add painless context examples for update and update-by-query (#37943) + + This commit improves the example docs for contexts in painless. + + relates #34829 + +commit 49bd8715ff63584b685c583644d5a5291f509edf +Author: Gordon Brown +Date: Mon Jan 28 14:09:12 2019 -0700 + + Inject Unfollow before Rollover and Shrink (#37625) + + We inject an Unfollow action before Shrink because the Shrink action + cannot be safely used on a following index, as it may not be fully + caught up with the leader index before the "original" following index is + deleted and replaced with a non-following Shrunken index. The Unfollow + action will verify that 1) the index is marked as "complete", and 2) all + operations up to this point have been replicated from the leader to the + follower before explicitly disconnecting the follower from the leader. + + Injecting an Unfollow action before the Rollover action is done mainly + as a convenience: This allow users to use the same lifecycle policy on + both the leader and follower cluster without having to explictly modify + the policy to unfollow the index, while doing what we expect users to + want in most cases. + +commit 19529da2db2c845a344b637301cd150e7cd7f656 +Author: Lisa Cawley +Date: Mon Jan 28 13:04:38 2019 -0800 + + [DOCS] Delayed data annotations (#37939) + +commit 557fcf915e987ac2afda8b7b79abbe00de2a2b25 +Author: Nhat Nguyen +Date: Mon Jan 28 15:25:56 2019 -0500 + + Wait for mapping in testReadRequestsReturnLatestMappingVersion (#37886) + + If the index request is executed before the mapping update is applied on + the IndexShard, the index request will perform a dynamic mapping update. + This mapping update will be timeout (i.e, ProcessClusterEventTimeoutException) + because the latch is not open. This leads to the failure of the index + request and the test. This commit makes sure the mapping is ready + before we execute the index request. + + Closes #37807 + +commit 6325e55dbfc494f2cb1e33411c91f60661136851 +Author: Ryan Ernst +Date: Mon Jan 28 11:56:36 2019 -0800 + + Use quotes in reproduce line task for vagrant failure (#37884) + + This commit wraps the packaging test task for reproducing a vagrant test + failure in quotes. The task names sometimes contain "#", which confuses + bash. + +commit 99b75a9bdf68f2629f7a4da5213511f1ca450655 +Author: Jake Landis +Date: Mon Jan 28 13:46:43 2019 -0600 + + deprecate types for watcher (#37594) + + This commit adds deprecation warnings for index actions + and search actions when executed via watcher. Unit and + integration tests updated accordingly. + + relates #35190 + +commit 9ceb218d8597ed7aa565ce2b0e1d8f10d137c96a +Author: Nhat Nguyen +Date: Sun Jan 27 21:50:46 2019 -0500 + + Adjust bwc version for put mapping requests + + Relates #37675 + +commit 0a850f032bb3093663ab39737c4c9d0748ff99d2 +Author: Luca Cavanna +Date: Mon Jan 28 16:36:39 2019 +0100 + + Handle deprecation warnings in a permissive manner + + Relates to #37290 + +commit 7e4c0e699171d8dd93b0bf0d2e6939231cd67683 +Author: Benjamin Trent +Date: Mon Jan 28 09:07:30 2019 -0600 + + ML: Adds set_upgrade_mode API endpoint (#37837) + + * ML: Add MlMetadata.upgrade_mode and API + + * Adding tests + + * Adding wait conditionals for the upgrade_mode call to return + + * Adding tests + + * adjusting format and tests + + * Adjusting wait conditions for api return and msgs + + * adjusting doc tests + + * adding upgrade mode tests to black list + +commit e401ab172491a64f0c97c9c4ebf1626e3b908bba +Author: Martijn van Groningen +Date: Mon Jan 28 15:02:50 2019 +0100 + + Handle deprecation warnings in a permissive manner. + + Closes #37920 + +commit 0d109396faac0dd41b482196c822a4b9886552a0 +Author: Armin Braun +Date: Mon Jan 28 14:13:02 2019 +0100 + + Increase Timeout in #testSnapshotCanceled (#37890) + + * The test failure reported in the issue looks like a mere timeout. Logging suggestst hat the snapshot completes/aborts correctly but the busy + loop polling the snapshot state times out too early. + * Closes #37888 + +commit 519423b8f39364c1ff78c4c9c98095e77572800d +Author: Luca Cavanna +Date: Mon Jan 28 14:07:12 2019 +0100 + + Mute failing full-cluster-restart tests + + Relates to #37920 + +commit 445db97867fd032aecfcd753ffb47a9aaa6ada93 +Author: Martijn van Groningen +Date: Mon Jan 28 13:55:53 2019 +0100 + + each full cluster restart round should use its own repository, + otherwise snapshots from e.g. 6.5.5 to current and 6.6.0 to current + full cluster restart round collides. + +commit a9adc16922c45d57f84c473658b4013e7d15131a +Author: Luca Cavanna +Date: Mon Jan 28 13:41:05 2019 +0100 + + Mute failing SearchQueryIT test + + Relates to #37814 + +commit 64b98db9736cc42a4a4ba80704684d0be468fe13 +Author: Alpar Torok +Date: Mon Jan 28 14:26:22 2019 +0200 + + Add an alias for :server:integTest so it runs as part of internalClusterTest (#37910) + +commit 194cdfe20815bdd99c45cf06882445f3333e2df9 +Author: Jason Tedor +Date: Mon Jan 28 07:11:51 2019 -0500 + + Sync retention leases on expiration (#37902) + + This commit introduces a sync of retention leases when a retention lease + expires. As expiration of retention leases is lazy, their expiration is + managed only when getting the current retention leases from the + replication tracker. At this point, we callback to our full retention + lease sync to sync and flush these on all shard copies. With this + change, replicas do not locally manage expiration of retention leases; + instead, that is done only on the primary. + +commit 758eb9d451eebd90d787c1e6ec70a49b4dd20c73 +Author: Tanguy Leroux +Date: Mon Jan 28 11:30:20 2019 +0100 + + Track accurate total hits in CloseIndexIT + + The test was not using the TRACK_TOTAL_HITS_ACCURATE and thus + encountered a different issue tracked in #37907. In the meanwhile + we can adapt the test to not fail anymore. + + Closes #37897 + +commit c0409fb9f03f14e29160a2c57cbf203e168f51aa +Author: David Kyle +Date: Mon Jan 28 10:00:59 2019 +0000 + + [ML] Marginal gains in slow multi node QA tests (#37825) + + Move 2 tests that are simple rest tests and out of the QA suite and cut the number + of post data calls in ForecastIT + +commit 2a610abef2b76314c750035a39c5accb984abb45 +Author: David Turner +Date: Mon Jan 28 09:45:55 2019 +0000 + + Document that auto_create_index is dynamic (#37903) + + We changed the `action.auto_create_index` setting to be a dynamic cluster-level + setting in #20274 but today the reference manual indicates that it is still a + static node-level setting. This commit addresses this, and clarifies the + semantics of patterns that may both permit and forbid the creation of certain + indices. + + Relates #7513 + +commit 57d321ed5f030cb6f17dba01b9d62e67133fc6d8 +Author: David Roberts +Date: Mon Jan 28 09:38:46 2019 +0000 + + [ML] Tighten up use of aliases rather than concrete indices (#37874) + + We have read and write aliases for the ML results indices. However, + the job still had methods that purported to reliably return the name + of the concrete results index being used by the job. After reindexing + prior to upgrade to 7.x this will be wrong, so the method has been + renamed and the comments made more explicit to say the returned index + name may not be the actual concrete index name for the lifetime of the + job. Additionally, the selection of indices when deleting the job + has been changed so that it works regardless of concrete index names. + + All these changes are nice-to-have for 6.7 and 7.0, but will become + critical if we add rolling results indices in the 7.x release stream + as 6.7 and 7.0 nodes may have to operate in a mixed version cluster + that includes a version that can roll results indices. + +commit a5f578f7eab25f19ce351669fd80068a5ed5a85e +Author: David Turner +Date: Mon Jan 28 09:29:07 2019 +0000 + + Fix duplicate note x-refs in mapping.asciidoc (#37904) + + The docs silently accept duplicate note markers (such as `<3>` here) but + formats them in an unexpected way. This change removes this duplication so that + the rendered documentation looks as intended. + +commit 4e1a779773311338c9a7a1336b1485fb2bd575a2 +Author: Martijn van Groningen +Date: Mon Jan 28 09:30:04 2019 +0100 + + Prepare ShardFollowNodeTask to bootstrap when it fall behind leader shard (#37562) + + * Changed `LuceneSnapshot` to throw an `OperationsMissingException` if the requested ops are missing. + * Changed the shard changes api to handle the `OperationsMissingException` and wrap the exception into `ResourceNotFound` exception and include metadata to indicate the requested range can no longer be retrieved. + * Changed `ShardFollowNodeTask` to handle this `ResourceNotFound` exception with the included metdata header. + + Relates to #35975 + +commit a056804831d8ea316213e8fb2ab723a532f4086c +Author: Jim Ferenczi +Date: Mon Jan 28 09:24:32 2019 +0100 + + Track total hits in tests that index more than 10,000 docs + + This change sets track_total_hits to true on a test that requires + to check the total hits of a query that can return more than 10,000 docs. + + Closes #37895 + +commit 290c6637c2475320d6206cdafdec026f731777f4 +Author: Dimitrios Liappis +Date: Mon Jan 28 10:01:26 2019 +0200 + + Refactor into appropriate uses of scheduleUnlessShuttingDown (#37709) + + Replace `threadPool().schedule()` / catch + `EsRejectedExecutionException` pattern with direct calls to + `ThreadPool#scheduleUnlessShuttingDown()`. + + Closes #36318 + +commit b1735aa93b557f173abf4c2337917cabdfcd30e0 +Author: Julie Tibshirani +Date: Sun Jan 27 16:02:22 2019 -0800 + + Support both typed and typeless 'get mapping' requests in the HLRC. (#37796) + + From previous PRs, we've already added support for include_type_name to + the get mapping API. We had also taken an approach to the HLRC where the + server-side `GetMappingResponse#fromXContent` could only handle typeless + input. + + This PR updates the HLRC for 'get mapping' to be in line with our new approach: + + * Add a typeless 'get mappings' method to the Java HLRC, that accepts new + client-side request and response objects. This new response only handles + typeless mapping definitions. + * Switch the old version of `GetMappingResponse` back to expecting typed + mappings, and deprecate the corresponding method on the HLRC. + + Finally, the PR also does some small, related clean-up around 'get field mappings'. + +commit 66ddd8d2f77d19334c36145e4bf4389de107cb1c +Author: Albert Zaharovits +Date: Sun Jan 27 23:07:32 2019 +0200 + + Create snapshot role (#35820) + + This commit introduces the `create_snapshot` cluster privilege and + the `snapshot_user` role. + This role is to be used by "cronable" tools that call the snapshot API + periodically without recurring to the `manage` cluster privilege. The + `create_snapshot` cluster privilege is much more limited compared to + the `manage` privilege. + + The `snapshot_user` role grants the privileges to view the metadata of + all indices (including restricted ones, i.e. .security). It obviously grants the + create snapshot privilege but the repository has to be created using another + role. In addition, it grants the privileges to (only) GET repositories and + snapshots, but not create and delete them. + + The role does not allow to create repositories. This distinction is important + because snapshotting equates to the `read` index privilege if the user has + control of the snapshot destination, but this is not the case in this instance, + because the role does not grant control over repository configuration. + +commit f24dce11228cc58ddbbbda5409b0bef701f71b4d +Author: Jason Tedor +Date: Sun Jan 27 08:16:14 2019 -0500 + + Fix newlines in retention lease sync action tests + + There is a method invocation here spanning multiple lines. This commit + breaks it up into a line per parameter as this is friendlier to future + changes and diffs. + +commit 3801925cf0f69f253a3e5b9af1d8889fca392313 +Author: Jason Tedor +Date: Sun Jan 27 08:13:47 2019 -0500 + + Copy retention leases under lock + + When adding a retention lease, we make a reference copy of the retention + leases under lock and then make a copy of that collection outside of the + lock. However, since we merely copied a reference to the retention + leases, after leaving a lock the underlying collection could change on + us. Rather, we want to copy these under lock. This commit adds a + dedicated method for doing this, asserts that we hold a lock when we use + this method, and changes adding a retention lease to use this method. + + This commit was intended to be included with #37398 but was pushed to + the wrong branch. + +commit 5fddb631a2b7e1be52233451b442eda70da88e3a +Author: Jason Tedor +Date: Sun Jan 27 07:49:56 2019 -0500 + + Introduce retention lease syncing (#37398) + + This commit introduces retention lease syncing from the primary to its + replicas when a new retention lease is added. A follow-up commit will + add a background sync of the retention leases as well so that renewed + retention leases are synced to replicas. + +commit cb134470c117794603fcdf6df223621c1869abd6 +Author: David Roberts +Date: Sun Jan 27 08:27:40 2019 +0000 + + [TEST] Fix MlMappingsUpgradeIT testMappingsUpgrade (#37769) + + Made the test tolerant to index upgrade being run + in between the old/mixed/upgraded portions. This + can occur because the rolling upgrade tests all + share the same indices. + + Fixes #37763 + +commit 780b4c72fe90c83ce08b7c5b96f70e91f53b1f73 +Author: Nhat Nguyen +Date: Sat Jan 26 22:01:30 2019 -0500 + + Make ChannelActionListener a top-level class (#37797) + + We start using this class more often. Let's make it a top-level class. + +commit f2c0c26d15335c92c47e52e9e06ea8b84c23e1cb +Author: David Roberts +Date: Sat Jan 26 20:19:57 2019 +0000 + + [ML] Adjust structure finder for Joda to Java time migration (#37306) + + The ML file structure finder has always reported both Joda + and Java time format strings. This change makes the Java time + format strings the ones that are incorporated into mappings + and ingest pipeline definitions. + + The BWC syntax of prepending "8" to these formats is used. + This will need to be removed once Java time format strings + become the default in Elasticsearch. + + This commit also removes direct imports of Joda classes in the + structure finder unit tests. Instead the core Joda BWC class + is used. + +commit afc60bb0e51aff57b8916401d693add63d56d0b8 +Author: Julie Tibshirani +Date: Fri Jan 25 18:09:34 2019 -0800 + + Mute DynamicMappingIT#testConflictingDynamicMappings + + Tracked in #37898. + +commit eb973a47441313e2699857fceba7a871fac7384f +Author: Tal Levy +Date: Fri Jan 25 17:29:04 2019 -0800 + + fix GeoHashGridTests precision parsing error + + Previously, a hardcoded precision value of 4 was + used by these tests resulting in no approximation + errors. Now that the precision is between 1-12, + precision values of 1 and 2 result in potential + bucketing errors. + + This commit adjusts the range to be 4-12. + + Fixes #37892. + +commit 58301ead6d23fe943a8ea02376ff5ef99768f42b +Author: Julie Tibshirani +Date: Fri Jan 25 17:12:14 2019 -0800 + + Mute IndexShardIT#testMaybeFlush + + Tracked in #37896. + +commit 23b0d9b3ed7f212e0a7c80c9eb08d215b4840784 +Author: Julie Tibshirani +Date: Fri Jan 25 16:50:39 2019 -0800 + + Mute RecoveryWhileUnderLoadIT#testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest + + Tracked in #37895. + +commit e41ccdc1a01278c0930500ec27b07353e1d50c81 +Author: Julie Tibshirani +Date: Fri Jan 25 16:15:14 2019 -0800 + + Mute GeoWKTShapeParserTests#testParseGeometryCollection + + Tracked in #37894. + +commit 24ff23812e730730408c72d44d9f7ea4928d3c91 +Author: Julie Tibshirani +Date: Fri Jan 25 14:38:00 2019 -0800 + + Mute TestClustersPluginIT and BuildExamplePluginsIT + + Tracked in #37889. + +commit 827ed1214634161742ec08c7b4dd1968dcca6b74 +Author: Julie Tibshirani +Date: Fri Jan 25 15:27:56 2019 -0800 + + Mute TasksIT#testTransportBulkTasks + + Tracked in #37893. + +commit a35701e43736aaac4af782cab677189a9abc13ff +Author: Sivagurunathan Velayutham +Date: Fri Jan 25 16:57:50 2019 -0600 + + Fix potential IllegalCapacityException in LLRC when selecting nodes (#37821) + +commit 7c130d235a4e516bf9e7feee496ad0a334e64074 +Author: Julie Tibshirani +Date: Fri Jan 25 14:55:45 2019 -0800 + + Mute CcrRepositoryIT#testFollowerMappingIsUpdated + + Tracked in #37887. + +commit 899dfc38bccce8da9c4e713702659ab813c8c321 +Author: Armin Braun +Date: Fri Jan 25 22:55:29 2019 +0100 + + Fix S3 Repository ITs When Docker is not Available (#37878) + + * Disable Minio fixture and tests that require it when fixtures are disabled or Docker is not available + * Relates #37852 + +commit 980c13d85cf43097c8b46da4d1d3450ded71ae42 +Author: Ryan Ernst +Date: Fri Jan 25 13:49:16 2019 -0800 + + Pass distribution type through to docs tests (#37885) + + This commit fixes the distribution flavor passed to the docs tests to be + the same as the distribution. These two values are now in sync (either + oss or default) for the docs tests. + +commit a4020f458779386c91b5c4bb49bc5496ca841d66 +Author: Julie Tibshirani +Date: Fri Jan 25 13:40:02 2019 -0800 + + Mute SharedClusterSnapshotRestoreIT#testSnapshotCanceledOnRemovedShard + + Tracked in #37888. + +commit d1ff450edc4cca7a6d6b682f5320d20d0130c8cb +Author: Marios Trivyzas +Date: Fri Jan 25 23:29:10 2019 +0200 + + SQL: Fix casting from date to numeric type to use millis (#37869) + + Previously casting from a DATE[TIME] type to a numeric (DOUBLE, LONG, + INT, etc. used seconds instead of the epoch millis. + + Fixes: #37655 + +commit eb7bf16427c6c9a07e248830afc53f6032cf30c0 +Author: Like +Date: Sat Jan 26 04:52:04 2019 +0800 + + Migrate o.e.i.r.RecoveryState to Writeable (#37380) + + Relates to #34389 + +commit 9e932f48699b4b331cfd34482d7058f46a683106 +Author: Benjamin Trent +Date: Fri Jan 25 13:57:41 2019 -0600 + + ML: removing unnecessary upgrade code (#37879) + +commit 5cd4dfb0e42b01fff2ce6e410339f4edca11be42 +Author: Nhat Nguyen +Date: Fri Jan 25 14:54:13 2019 -0500 + + Relax cluster metadata version check (#37834) + + If the in_sync_allocations of index-1 or index-2 is changed, the + metadata version will be increased. This leads to the failure in + the metadata version checks. We need to relax them. + + Closes #37820 + +commit 455f223c3a8526d7462159ab362754c890d2af6f +Author: Julie Tibshirani +Date: Fri Jan 25 11:11:46 2019 -0800 + + Mute TransformIntegrationTests#testSearchTransform + + Tracked in #37882. + +commit f1e71be8b24f65bc9fb340f269436acab3642366 +Author: Yuri Astrakhan +Date: Fri Jan 25 13:37:24 2019 -0500 + + Refactored GeoHashGrid unit tests (#37832) + + * Refactored GeoHashGrid unit tests + + This change allows other grid aggregations to reuse the same tests. + + The change mostly just moves code to the base classes, trying to + keep changes to a bare minimum. + + * rename createInternalGeoHashGridBucket to createInternalGeoGridBucket + + * indentation + +commit afd4618851813d15584f1aac656fab4d694fbeb6 +Author: Zachary Tong +Date: Fri Jan 25 12:33:42 2019 -0500 + + Fixes for a few randomized agg tests that fail hasValue() checks + + Closes #37743 + Closes #37873 + +commit 68149b6058a720ec891974e8a6982c5e480fb9f1 +Author: Igor Motov +Date: Fri Jan 25 11:37:27 2019 -0500 + + Geo: replace intermediate geo objects with libs/geo (#37721) + + Replaces intermediate geo objects built by ShapeBuilders with + objects from the libs/geo hierarchy. This should allow us to build + all geo functionality around a single hierarchy. + + Follow up for #35320 + +commit e88ae99340505b607fb32b1a1509ff0afb6f3651 +Author: Ryan Ernst +Date: Fri Jan 25 08:11:48 2019 -0800 + + Remove NOREPLACE for /etc/elasticsearch in rpm and deb (#37839) + + The /etc/elasticsearch directory is currently configured as a config + file with noreplace. However, the directory itself is not config, and + can lead to an entire /etc/elasticsearch.rpmsave directory in some + situations. This commit fixes the ospackage config to not specify those + file bits for the directory itself, but only the files underneath it. + +commit acc3cae40c21f1807b1cbbe4663a8e7a432744da +Author: Ryan Ernst +Date: Fri Jan 25 08:07:49 2019 -0800 + + Remove "reinstall" packaging tests (#37851) + + The packaging tests currently have a test which installs elasticsearch, + removes it, modifies ownership of /etc/elasticsearch, and + reinstalls. It then checks that the /etc/elasticsearch directory has + ownership that the package expects. But the recursive change touches + files not owned by the package. In the past this worked because we did a + recursive ownership change within the package postinst. However, that + was recently removed, and thus this test no longer makes sense. + +commit a644bc095c347b9c823b880f9b7ac12dbaec8d47 +Author: Tanguy Leroux +Date: Fri Jan 25 16:51:53 2019 +0100 + + Add unit tests for ShardStateAction's ShardStartedClusterStateTaskExecutor (#37756) + +commit dfecb256cb566de2913cd00a040e289f80040428 +Author: Martijn Laarman +Date: Fri Jan 25 16:44:33 2019 +0100 + + Exit batch files explictly using ERRORLEVEL (#29583) + + * Exit batch files explictly using ERRORLEVEL + + This makes sure the exit code is preserved when calling the batch + files from different contexts other than DOS + + Fixes #29582 + + This also fixes specific error codes being masked by an explict + + exit /b 1 + + causing the useful exitcodes from ExitCodes to be lost. + + * fix line breaks for calling cli to match the bash scripts + + * indent size of bash files is 2, make sure editorconfig does the same for bat files + + * update indenting to match bash files + + * update elasticsearch-keystore.bat indenting + + * Update elasticsearch-node.bat to exit outside of endlocal + +commit f1f54e0f6137e55c403da2bb25e27b98c38f2ade +Author: Tanguy Leroux +Date: Fri Jan 25 16:31:26 2019 +0100 + + TransportUnfollowAction should increase settings version (#37859) + + The TransportUnfollowAction updates the index settings but does not + increase the settings version to reflect that change. + + This issue has been caught while working on the replication of closed + indices (#33888). The IndexFollowingIT.testUnfollowIndex() started to + fail and this specific assertion tripped. It does not happen on master + branch today because index metadata for closed indices are never + updated in IndexService instances, but this is something that is going + to change with the replication of closed indices. + +commit 85acc11ef7a183b97eebd35015039e7a5326bb14 +Author: Przemyslaw Gomulka +Date: Fri Jan 25 16:26:16 2019 +0100 + + AsyncTwoPhaseIndexerTests race condition fixed (#37830) + + The unlucky timing can cause this test to fail when the indexing is triggered from `maybeTriggerAsyncJob`. As this is asynchronous, in can finish quicker then the test stepping over to next assertion + The introduced barrier solves the problem + closes #37695 + +commit 27c3fb8e0d8462e359418293c13818be101b4eb9 +Author: Vishnu Gt +Date: Fri Jan 25 20:26:34 2019 +0530 + + Do not allow negative variances (#37384) + + Due to floating point error, it was possible for variances to become negative which should never happen. This bugfix sets variance to zero if it becomes negative as a result of fp error. + +commit ef8dd12c6d87f40e2c41da274bbb08136ae164e4 +Author: Tanguy Leroux +Date: Fri Jan 25 15:09:27 2019 +0100 + + Limit number of documents indexed in CloseIndexIT test + + This test indexes an unlimited number of documents, this commit + reduces this number to 25K and also tracks exact number of hits + when counting the docs. + +commit b4b4cd6ebd09c88d857b50f43b9b7b3331ead5d3 +Author: Christoph Büscher +Date: Fri Jan 25 14:23:02 2019 +0100 + + Clean codebase from empty statements (#37822) + + * Remove empty statements + + There are a couple of instances of undocumented empty statements all across the + code base. While they are mostly harmless, they make the code hard to read and + are potentially error-prone. Removing most of these instances and marking blocks + that look empty by intention as such. + + * Change test, slightly more verbose but less confusing + +commit 49073dd2f680c0fde81c810b8da28a0af33f63e5 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Fri Jan 25 14:22:48 2019 +0100 + + Fail start on invalid index metadata (#37748) + + Node started with node.data=false and node.master=false can no longer + start if they have index metadata. This avoids resurrecting old indexes + into the cluster and ensures metadata is cleaned out before + re-purposing a node that was previously master or data node. + + Issue #27073 + +commit deafce1acda47f5d59de8ec2ec55b43ba986586f +Author: David Roberts +Date: Fri Jan 25 13:15:35 2019 +0000 + + [ML] No need to add state doc mapping on job open in 7.x (#37759) + + When upgrading from 5.4 to 5.5 to 6.7 (inclusive) it was + necessary to ensure there was a mapping for type "doc" on + the ML state index before opening a job. This was because + 5.4 created a multi-type ML state index. + + In version 7.x we can be sure that any such 5.4 index is no + longer in use. It would have had to be reindexed into the + 6.x index format prior to the upgrade to version 7.x. + +commit cb451edb0147db2d8d643ccc21eaa33413831e1b +Author: Jim Ferenczi +Date: Fri Jan 25 14:00:39 2019 +0100 + + Allow nested fields in the composite aggregation (#37178) + + This changes adds the support to handle `nested` fields in the `composite` + aggregation. A `nested` aggregation can be used as parent of a `composite` + aggregation in order to target `nested` fields in the `sources`. + + Closes #28611 + +commit 9e350d027e0a908b7e8a54fbd73141669cf48641 +Author: Alexander Reelsen +Date: Fri Jan 25 13:50:19 2019 +0100 + + Add BWC compatible processing to ingest date processors (#37407) + + The ingest date processor is currently only able to parse joda formats. + However it is not using the existing elasticsearch classes but access + joda directly. This means that our existing BWC layer does not notify + the user about deprecated formats. This commit switches to use the + exising Elasticsearch Joda methods to acquire a date format, that + includes the BWC check and the ability to parse java 8 dates. + + The date parsing in ingest has also another extra feature, that the + fallback year, when a date format without a year is used, is the current + year, and not 1970 like usual. This is currently not properly supported + in the DateFormatter class. As this is the only case for this feature + and java time can take care of this using the toZonedDateTime() method, + a workaround just for the joda time parser has been created, that can be + removed soon again from 7.0. + +commit 787acb14b91b7065c4ed80167fb4bdb6cc444c86 +Author: Jim Ferenczi +Date: Fri Jan 25 13:45:39 2019 +0100 + + Track total hits up to 10,000 by default (#37466) + + This commit changes the default for the `track_total_hits` option of the search request + to `10,000`. This means that by default search requests will accurately track the total hit count + up to `10,000` documents, requests that match more than this value will set the `"total.relation"` + to `"gte"` (e.g. greater than or equals) and the `"total.value"` to `10,000` in the search response. + Scroll queries are not impacted, they will continue to count the total hits accurately. + The default is set back to `true` (accurate hit count) if `rest_total_hits_as_int` is set in the search request. + I choose `10,000` as the default because that's also the number we use to limit pagination. This means that users will be able to know how far they can jump (up to 10,000) even if the total number of hits is not accurate. + + Closes #33028 + +commit e7f0adb0c5ab49ba64c177cf73e4fa4a9b13b91b +Author: Alpar Torok +Date: Fri Jan 25 14:15:09 2019 +0200 + + Add path sensitivity annotations (#37762) + + The plugin builder plugin generates warnings for these. + There's no immediate impact as we don't have a shared build cache. + +commit 7b516f99b975eaf8b8d52d179557e0b347ada00b +Author: Alpar Torok +Date: Fri Jan 25 14:14:12 2019 +0200 + + Resolve BWC dependencies too when resolving deps (#37412) + + This will allow those to be cached too leading to faster builds and + fewer opportunity for network failures + +commit 70af3c79831512f3f4edfb0bf06c733a09a82eb7 +Author: Mayya Sharipova +Date: Fri Jan 25 07:13:46 2019 -0500 + + Correct deprec log in RestGetFieldMappingAction (#37843) + + * Correct deprec log in RestGetFieldMappingAction + + Correct a class used for deprecation logging in + RestGetFieldMappingAction + + * Correct deprec log in RestCreateIndexAction + + Correct a class used for deprecation logging in + RestCreateIndexAction + +commit 97e89ad0bc6f1d4bb13571fdca05280d2369e93f +Author: Alpar Torok +Date: Fri Jan 25 14:13:34 2019 +0200 + + Testclusters logging improovements (#37354) + + - Cluster logs are only indented as node name is already in the logs + - silence logging on shutdown + - have fully qualified name as node and cluster name + +commit be6bdab346fa5058361ed18cbd3cc29ff9052b4f +Author: Armin Braun +Date: Fri Jan 25 12:56:40 2019 +0100 + + Use TestFixturesPlugin to Run Minio in Tests (#37852) + + * Use TestFixturesPlugin to Run Minio in Tests + + * Closes #37680 + * Closes #37783 + +commit a3baa8f5effa7c8466151b72758f1e574896a61a +Author: Tanguy Leroux +Date: Fri Jan 25 11:27:27 2019 +0100 + + Freezing an index should increase its index settings version (#37813) + + When an index is frozen, two index settings are updated (index.frozen and + index.search.throttled) but the settings version is left unchanged and does + not reflect the settings update. This commit change the + TransportFreezeIndexAction so that it also increases the settings version + when an index is frozen/unfrozen. + + This issue has been caught while working on the replication of closed + indices (#3388) in which index metadata for a closed index are updated + to frozen metadata and this specific assertion tripped. + +commit 9e7fd8caed6bd67e0268f7cc71736f7dbaa0cf5c +Author: Andrey Ershov +Date: Fri Jan 25 11:17:09 2019 +0100 + + Migrate ZenDiscoveryIT to Zen2 (#37465) + + ZenDiscoveryIT contained 5 tests. 3 run without changes, testNodeRejectsClusterStateWithWrongMasterNode removed, testHandleNodeJoin_incompatibleClusterState changed. + +commit 170d7413d099a624d6d7ba7448ecd1b228d0f64c +Author: David Roberts +Date: Fri Jan 25 09:29:53 2019 +0000 + + [ML] Fix gaps in reserved roles tests (#37772) + + Some of our newer endpoints and indices were missing from + the tests. + +commit 7692b607b930f632d6ea5e5e0eda55fcb57cfc45 +Author: Armin Braun +Date: Fri Jan 25 09:38:29 2019 +0100 + + Fix ClusterDisruptionIT#testAckedIndexing (#37853) + + * Stop threads before logging the list of exceptions + * For the broken case of concurrent iteration in the finally block and the threads not having shut down, + use `CopyOnWriteArrayList` to have concurrency safe iteration + * Closes #37810 + +commit 5a9dadb3ff287893446d39bd9865269583cb8971 +Author: Martijn van Groningen +Date: Fri Jan 25 09:18:42 2019 +0100 + + changed versionAdded now that #37767 is backedported + +commit 1151f3b3ffaa3c3d4024f3b19b816a3ae916d00a +Author: Martijn van Groningen +Date: Fri Jan 25 08:53:18 2019 +0100 + + Fail with a dedicated exception if remote connection is missing or (#37767) + + or connectivity to the remote connection is failing. + + Relates to #37681 + +commit df8fa9781ee4dc029065a7daa4d29ab6a9e5a200 +Author: Ricardo Ferreira +Date: Fri Jan 25 07:35:19 2019 +0000 + + Remove Abstract Component (#35898) + + TransportAction and BaseRestHandler now no longer extends AbstractComponent. The AbstractComponent no longer has usages so it was deleted. + + Closes #34488 + +commit 03690d12b24d742e26149ef09a881f2f37375865 +Author: Tim Vernum +Date: Fri Jan 25 15:46:39 2019 +1100 + + Remove TLS 1.0 as a default SSL protocol (#37512) + + The default value for ssl.supported_protocols no longer includes TLSv1 + as this is an old protocol with known security issues. + Administrators can enable TLSv1.0 support by configuring the + appropriate `ssl.supported_protocols` setting, for example: + + xpack.security.http.ssl.supported_protocols: ["TLSv1.2","TLSv1.1","TLSv1"] + + Relates: #36021 + +commit 6a13a252e9d0eadc7bd099b3415d503beb3b7a7e +Author: Yuri Astrakhan +Date: Thu Jan 24 23:45:18 2019 -0500 + + Abstract GeoHashGridAggregatorFactory creation, renamed geohash -> hash (#37836) + + * Delegate `new GeoHashGridAggregatorFactory(...)` inside the `GeoGridAggregationBuilder` to the child classes. + * Rename all `geohash...` to `hash...` + +commit 3ccd4887556726f13ef07ad6ab1733a010018119 +Author: Nhat Nguyen +Date: Thu Jan 24 21:48:50 2019 -0500 + + Remove testMappingsPropagatedToMasterNodeImmediately + + This test is obsolete since #31140 where an index request with dynamic + mapping update no longer requires acking. + + Closes #37816 + +commit a30ce6a00a9aa4c82320c9d8f4427652ce9078fd +Author: Mayya Sharipova +Date: Thu Jan 24 19:18:48 2019 -0500 + + Rename feature, feature_vector and feature_query (#37794) + + Ranaming as follows: + feature -> rank_feature + feature_vector -> rank_features + feature query -> rank_feature query + + Ranaming is done to distinguish from other vector types. + + Closes #36723 + +commit 9d87ca567a9adf718ed3583d7423e42d42189a88 +Author: Boaz Leskes +Date: Thu Jan 24 23:40:12 2019 +0100 + + 300_sequence_numbers should not rely on 7.0 total hits structure + +commit 4313c2d4a77818094b66791f6afa69238ae9f5f8 +Author: Peter Dyson +Date: Fri Jan 25 08:39:58 2019 +1000 + + [DOCS] More info on disabling swap (#37248) + + * [DOCS] More info on disabling swap, inform that no Elasticsearch restart is required. + +commit 0f3c542850a477560d72935170808a6130cd3aca +Author: Lee Hinman +Date: Thu Jan 24 15:31:31 2019 -0700 + + Deprecate xpack.watcher.history.cleaner_service.enabled (#37782) + + This deprecates the `xpack.watcher.history.cleaner_service.enabled` setting, + since all newly created `.watch-history` indices in 7.0 will use ILM to manage + their retention. + + In 8.0 the setting itself and cleanup actions will be removed. + + Resolves #32041 + +commit d473bcda8d9fcb6330efe333f662f1c214ab631c +Author: Julie Tibshirani +Date: Thu Jan 24 14:08:58 2019 -0800 + + Remove outdated callouts from the 'create index' HLRC docs + +commit 967de04257fb57459e21625de65e81d66705fc2e +Author: Christoph Büscher +Date: Thu Jan 24 22:54:55 2019 +0100 + + Uppercasing some docs section title (#37781) + + Section titles are mostly uppercase, only a few cases where query DSL parameters + or Java method names are used as the title they should be lowercased. + +commit e1d8df4ffae3d25a8ddd73a04234d5887cda45c0 +Author: Julie Tibshirani +Date: Thu Jan 24 13:17:47 2019 -0800 + + Deprecate types in create index requests. (#37134) + + From #29453 and #37285, the include_type_name parameter was already present and defaulted to false. This PR makes the following updates: + * Add deprecation warnings to RestCreateIndexAction, plus tests in RestCreateIndexActionTests. + * Add a typeless 'create index' method to the Java HLRC, and deprecate the old typed version. To do this cleanly, I created new CreateIndexRequest and CreateIndexResponse objects that differ from the existing server ones. + +commit af2f4c8f73673ecba663c6bfa52409d9790861ac +Author: Boaz Leskes +Date: Thu Jan 24 12:15:05 2019 +0100 + + enable bwc tests and bump versions after backporting https://github.com/elastic/elasticsearch/pull/37639 + +commit 864e46551536605a8e76b401068791069912382a +Author: Nhat Nguyen +Date: Thu Jan 24 13:09:43 2019 -0500 + + Adjust minRetainedSeqNo asssertion in CombinedDeletionPolicyTests + + In these tests, we initialize the retained_seq_no with NO_OPS_PERFORMED, + thus we should verify that the min of the retained_seq_no is at least + NO_OPS_PERFORMED not 0. + + Closes #35994 + +commit 497468400365264939ac1f841bf9e619b4d2070e +Author: Andrey Ershov +Date: Thu Jan 24 19:25:55 2019 +0100 + + Add tool elasticsearch-node unsafe-bootstrap (#37696) + + elasticsearch-node tool helps to restore cluster if half or more of + master eligible nodes are lost. Of course, all bets are off, regarding + data consistency. + + There are two parts of the tool: unsafe-bootstrap to be used when there + is still at least one master-eligible node alive and detach-cluster, + when there are no master-eligible nodes left. + This commit implements the first part. + + Docs for the tool will be added separately as a part of #37812. + +commit 289106a578284aa40f1026c09dd8fa4ff4ea23ee +Author: Tal Levy +Date: Thu Jan 24 10:12:14 2019 -0800 + + Refactor GeoHashGrid to be abstract and re-usable (#37742) + + This change split out all the specific GeoHash + classes for the geohash_grid aggregation into + abstract GeoGrid classes that can be re-used for + specific hashing types, like `geohash` + +commit 33cac52b2c1919048e5e0772f1cb8911e88a456c +Author: Lisa Cawley +Date: Thu Jan 24 09:14:08 2019 -0800 + + [DOCS] Remove beta tag from metricbeat monitoring (#37791) + +commit 76fb57356956bd8d9d1e8d73241b8abb3048944d +Author: Nhat Nguyen +Date: Thu Jan 24 12:13:00 2019 -0500 + + Do not allow put mapping on follower (#37675) + + Today, the mapping on the follower is managed and replicated from its + leader index by the ShardFollowTask. Thus, we should prevent users + from modifying the mapping on the follower indices. + + Relates #30086 + +commit 187b233571073e27f4ba04d0b05595cc1e7b53ce +Author: David Turner +Date: Thu Jan 24 11:57:37 2019 +0000 + + Read m_m_n from cluster states from 6.7 + + This completes the BWC serialisation changes required for a 6.7 master to + inform other nodes of the node-level value of the `minimum_master_nodes` + setting. + + Relates #37701, #37811 + +commit 0e36adc35fafbb0a3fb5bfa459f9cb7fac520785 +Author: David Roberts +Date: Thu Jan 24 14:08:30 2019 +0000 + + Mute SimpleClusterStateIT testMetadataVersion + + Due to https://github.com/elastic/elasticsearch/issues/37820 + +commit 74b6f308e9b157db947b961d536e6d0bd3b18dd5 +Author: Marios Trivyzas +Date: Thu Jan 24 18:40:20 2019 +0200 + + SQL: Fix issue with complex expression as args of PERCENTILE/_RANK (#37102) + + When the arguements of PERCENTILE and PERCENTILE_RANK can be folded, + the `ConstantFolding` rule kicks in and calls the `replaceChildren()` + method on `InnerAggregate` which is created from the aggregation rules + of the `Optimizerz. `InnerAggregate` in turn, cannot implement the method + as the logic of creating a new `InnerAggregate` instance from a list of + `Expression`s resides in the Optimizer. So, instead, `ConstantFolding` + should be applied before any of the aggregations related rules. + + Fixes: #37099 + +commit f6b6f927ec83b0067d330d606dcf869db97d7029 +Author: Alpar Torok +Date: Thu Jan 24 17:33:08 2019 +0200 + + Fail randomized testing tasks if no tests ran (#37764) + + Reverts #36259 in part to make randomized test fail if no tests are ran. + This is useful when filtering tests as it's easy to make a typo and + think the test ran trough successfully. + +commit 37768b7eac24ad226dc0e4b0d34614d98b33fcd2 +Author: Alpar Torok +Date: Thu Jan 24 17:30:50 2019 +0200 + + Testing conventions now checks for tests in main (#37321) + + * Testing conventions now checks for tests in main + + This is the last outstanding feature of the old NamingConventionsTask, + so time to remove it. + + * PR review + +commit e7aa7e909a370a54613e86592d58219c7326faaf +Author: Alpar Torok +Date: Thu Jan 24 17:30:24 2019 +0200 + + Switch testclusters to use nio (#37365) + +commit ddee1926f977372b307f15adafda043fe5fd6fa7 +Author: niloct +Date: Thu Jan 24 13:24:42 2019 -0200 + + Update update-by-query.asciidoc (#37555) + + Similar fix as #37370. + +commit 4e08cca6bc1e63cb72ba5301f50decb4ce52a615 +Author: Alpar Torok +Date: Thu Jan 24 17:26:42 2019 +0200 + + Ground work to start up the docker image in the build (#37754) + + This change adds a docker compose configuration that's used with + the `elasticsearch.test.fixtures` plugin to start up the image + and check that the TCP ports are up. + + We can build on this to add other checks for culster health, + run REST tests, etc. + + We can add multiple containers and configurations to the compose + file (e.x. test different env vars) and form clusters. + +commit bd02ca4b7b066de1e77cf9246c5a1674af1ea71a +Author: David Roberts +Date: Thu Jan 24 15:17:13 2019 +0000 + + Mute NoMasterNodeIT testNoMasterActionsWriteMasterBlock + + Due to https://github.com/elastic/elasticsearch/issues/37823 + +commit 265710e6582ec7943af56b47203a1c5dbaf19dc4 +Author: Ioannis Kakavas +Date: Thu Jan 24 17:05:01 2019 +0200 + + Better msg on unmapped principal attribute (#37805) + + When we can't map the principal attribute from the configured SAML + attribute in the realm settings, we can't complete the + authentication. We return an error to the user indicating this and + we present them with a list of attributes we did get from the SAML + response to point out that the expected one was not part of that + list. This list will never contain the NameIDs though as they are + not part of the SAMLAttribute list. So we might have a NameID but + just with a different format. + +commit a6abb28abf844fa46ca966dcbfcee99ee019f7b6 +Author: Nhat Nguyen +Date: Thu Jan 24 10:02:48 2019 -0500 + + Fix InternalEngineTests#assertOpsOnPrimary (#37746) + + The assertion `assertOpsOnPrimary` does not store seq_no and primary + term of successful deletes to the `lastOpSeqNo` and `lastOpTerm`. This + leads to failures of the subsequence CAS deletes or indexes with seq_no + and term. Moreover, this assertion trips a translog assertion because it + bumps the primary term of some operations but not the primary term of + the engine. + + Relates #36467 + Closes #37684 + +commit a81931bb2a9c6e2ee260dc8515b8c3b2b2ea3651 +Author: David Roberts +Date: Thu Jan 24 12:54:23 2019 +0000 + + Mute DynamicMappingIT testMappingsPropagatedToMasterNodeImmediately + + Due to https://github.com/elastic/elasticsearch/issues/37816 + +commit 163a27b93c6e08c84718c3c6a509c68db3bf9937 +Author: Andrei Stefan +Date: Thu Jan 24 15:40:51 2019 +0200 + + SQL: Fix BasicFormatter NPE (#37804) + +commit 7517e3a7bddfc211e3c55fb4463c532d8c330a47 +Author: Jason Tedor +Date: Thu Jan 24 08:39:24 2019 -0500 + + Optimize warning header de-duplication (#37725) + + Now that warning headers no longer contain a timestamp of when the + warning was generated, we no longer need to extract the warning value + from the warning to determine whether or not the warning value is + duplicated. Instead, we can compare strings directly. + + Further, when de-duplicating warning headers, are constantly rebuilding + sets. Instead of doing that, we can carry about the set with us and + rebuild it if we find a new warning value. + + This commit applies both of these optimizations. + +commit feab59df039d99a177bee4e4583be0cd8346c2c1 +Author: Yannick Welsch +Date: Thu Jan 24 14:09:03 2019 +0100 + + Bubble exceptions up in ClusterApplierService (#37729) + + Exceptions thrown by the cluster applier service's settings and cluster appliers are bubbled up, and + block the state from being applied instead of silently being ignored. In combination with the cluster + state publishing lag detector, this will throw a node out of the cluster that can't properly apply + cluster state updates. + +commit 93579293095374457c84494888b95b761cb7bd62 +Author: Marios Trivyzas +Date: Thu Jan 24 15:03:49 2019 +0200 + + SQL: Improve handling of invalid args for PERCENTILE/PERCENTILE_RANK (#37803) + + Improve the Exception and the error message returned when 2nd argument + of PERCENTILE and PERCENTILE_RANK is not a constant. + +commit c7b16162ae456ca6d06cafc2435cfff1d00c1998 +Author: Simon Willnauer +Date: Thu Jan 24 13:52:22 2019 +0100 + + Remove unused ThreadBarrier class (#37666) + + This class is pretty complex and only used in a test where we can simply + fail the test with an assertion error. + +commit 20533c59909fd6008c8d3e1d3789ec884546a853 +Author: Yulong +Date: Thu Jan 24 20:12:32 2019 +0800 + + Add built-in user and role for code plugin (#37030) + + * Add built-in roles for code plugin + + * Fix rest-client get-roles test count + + * Fix broken test + +commit bc201427912a4d33b3aab91abb418aa0004dea2a +Author: Alpar Torok +Date: Thu Jan 24 13:47:21 2019 +0200 + + Consolidate testclusters tests into a single project (#37362) + +commit 2bf269e628f5c008b851e7fed408b2814fb8ca4c +Author: Yannick Welsch +Date: Thu Jan 24 12:43:48 2019 +0100 + + Fix docs for MappingUpdatedAction + + Follow-up to #31140 + +commit f707fa9e0af4264f2aebb24b43543d15c7a8875f +Author: Marios Trivyzas +Date: Thu Jan 24 13:41:58 2019 +0200 + + SQL: Introduce SQL DATE data type (#37693) + + * SQL: Introduce SQL DATE data type + + Support ANSI SQL's DATE type by introducing a runtime-only + ES SQL date type. + + Closes: #37340 + +commit b6317ed70bd7772b17e6c8b038b06d8575ba9a2a +Author: Boaz Leskes +Date: Thu Jan 24 11:53:49 2019 +0100 + + disabling bwc test while backporting https://github.com/elastic/elasticsearch/pull/37639 + +commit bcf5a4ca479cedd7bf146dd4c8eedaddbba4de7d +Author: David Roberts +Date: Thu Jan 24 10:58:02 2019 +0000 + + Mute ClusterDisruptionIT testAckedIndexing + + Due to https://github.com/elastic/elasticsearch/issues/37810 + +commit 64adb5ad5bdeb68490f6664c6907bfe30cf02d78 +Author: Yannick Welsch +Date: Thu Jan 24 11:39:46 2019 +0100 + + Set acking timeout to 0 on dynamic mapping update (#31140) + + As acking can fail for any reason (unrelated node being too slow, node disconnecting), it should not + be required for acking to succeed in order for index requests with dynamic mapping updates to + successfully complete. + + Relates to #30672 and Closes #30844 + +commit b6936e3c1e00af50393c9be15f966cde333f46f4 +Author: Albert Zaharovits +Date: Thu Jan 24 12:36:10 2019 +0200 + + Remove index audit output type (#37707) + + This commit removes the Index Audit Output type, following its deprecation + in 6.7 by 8765a31d4e6770. It also adds the migration notice (settings notice). + + In general, the problem with the index audit output is that event indexing + can be slower than the rate with which audit events are generated, + especially during the daily rollovers or the rolling cluster upgrades. + In this situation audit events will be lost which is a terrible failure situation + for an audit system. + Besides of the settings under the `xpack.security.audit.index` namespace, the + `xpack.security.audit.outputs` setting has also been deprecated and will be + removed in 7. Although explicitly configuring the logfile output does not touch + any deprecation bits, this setting is made redundant in 7 so this PR deprecates + it as well. + + Relates #29881 + +commit f12bfb4684a1ec9cfc4d99f5e7becaaa920022c6 +Author: David Roberts +Date: Thu Jan 24 09:58:50 2019 +0000 + + Mute FollowerFailOverIT testReadRequestsReturnsLatestMappingVersion + + Due to https://github.com/elastic/elasticsearch/issues/37807 + +commit e1226f69b7648dc50204afd641969a3b9f283a83 +Author: David Kyle +Date: Thu Jan 24 09:18:48 2019 +0000 + + [ML] Increase close job timeout and lower the max number (#37770) + +commit 36889e8a2f8320a4e8a7528b7247f087346b1a85 +Author: Armin Braun +Date: Thu Jan 24 10:11:18 2019 +0100 + + Remove Custom Listeners from SnapshotsService (#37629) + + * Remove Custom Listeners from SnapshotsService + + Motivations: + * Shorten the code some more + * Use ActionListener#wrap to get easy to reason about behavior in failure scenarios + * Remove duplication in the logic of handling snapshot completion listeners (listeners removing themselves and comparing snapshots to their targets) + * Also here, move all listener handling into `SnapshotsService` and remove custom listener class by putting listeners in a map + +commit bdef2ab8c047e31d84e7fdbe9c843e57802a6291 +Author: David Turner +Date: Thu Jan 24 08:57:40 2019 +0000 + + Use m_m_nodes from Zen1 master for Zen2 bootstrap (#37701) + + Today we support a smooth rolling upgrade from Zen1 to Zen2 by automatically + bootstrapping the cluster once all the Zen1 nodes have left, as long as the + `minimum_master_nodes` count is satisfied. However this means that Zen2 nodes + also require the `minimum_master_nodes` setting for this one specific and + transient situation. + + Since nodes only perform this automatic bootstrapping if they previously + belonged to a Zen1 cluster, they can keep track of the `minimum_master_nodes` + setting from the previous master instead of requiring it to be set on the Zen2 + node. + +commit 2908ca1b3509ed755cf9d0242d54349c60d4fa8f +Author: Martijn van Groningen +Date: Thu Jan 24 08:50:23 2019 +0100 + + Fix index filtering in follow info api. (#37752) + + The filtering by follower index was completely broken. + Also the wrong persistent tasks were selected, causing the + wrong status to be reported. + + Closes #37738 + +commit d9d13f34146a2ee29656b52a440fa3be5b8c2cf6 +Author: Ryan Ernst +Date: Wed Jan 23 23:41:21 2019 -0800 + + Use project dependency instead of substitutions for distributions (#37730) + + Currently integration tests which use either bwc snapshot versions or + the current version of elasticsearch depend on project substitutions to + link to the build of those artifacts. Likewise, vagrant tests use + dependency substitutions to get to bwc snapshots of rpm and debs. + This commit changes those to depend on the relevant project/configuration + and removes the dependency substitutions for distributions we do not + publish. + +commit 04c64147bdd4137fb3d1804b810bf64a95969f9f +Author: Michael Basnight +Date: Wed Jan 23 22:14:01 2019 -0600 + + Update authenticate to allow unknown fields (#37713) + + AuthenticateResponse did not allow unknown fields. This commit fixes the + test and ConstructingObjectParser such that it does now allow unknown + fields. + + Relates #36938 + +commit 944972a2494c7c2f9f8a09f30dc72914b8e1f435 +Author: Michael Basnight +Date: Wed Jan 23 22:13:16 2019 -0600 + + Deprecate HLRC EmptyResponse used by security (#37540) + + The EmptyResponse is essentially the same as returning a boolean, which + is done in other places. This commit deprecates all the existing + EmptyResponse methods and creates new boolean methods that have method + params reordered so they can exist with the deprecated methods. A + followup PR in master will remove the existing deprecated methods, fix + the parameter ordering and deprecate the incorrectly ordered parameter + methods. + + Relates #36938 + +commit 49b8b07758d504ce9183022c95cfb3b01df981df +Author: Ryan Ernst +Date: Wed Jan 23 17:39:13 2019 -0800 + + Use explicit version for build-tools in example plugin integ tests (#37792) + + The example plugins are currently built within the build-tools integ + tests as a means to ensure the gradle plugin works for external plugin + builds. These tests generate a dummy build.gradle, and a dummy local + maven repository to find the local builds dependencies in. Currently + that build-tools dependency uses "+" as the version. However, this + allows gradle to find the "latest" version, and unfortunately gradle has + its own plugin repository which is apparently connected to jcenter. This + recently triggered a flood of CI failures when jcenter suddenly pulled + alpha2, and all builds started trying to use that instead of the locally + built build-tools. This commit uses the explicit version of build-tools + that was build locally, which will cause resolution to stop when the + local repo is first checked. + +commit fdb66039d4d583f20afcacd6e10ef9f1b0464c10 +Author: Mayya Sharipova +Date: Wed Jan 23 14:28:20 2019 -0500 + + Change `rational` to `saturation` in script_score (#37766) + + This change of the function name is necessary for conformity + with feature queries. + + Closes #37714 + +commit c8565fe6928659528b642e3e922d0c7f385e877b +Author: Mayya Sharipova +Date: Wed Jan 23 14:24:35 2019 -0500 + + Deprecate types in get field mapping API (#37667) + + - Add deprecation warning to RestGetFieldMappingAction + - Add two new java HRLC classes GetFieldMappingsRequest and + GetFieldMappingsResponse. These classes use new typeless forms + of a request and response, and differ in that from the server + versions. + + Relates to #35190 + +commit f45b5fedb56b315a7daf24719b671603b91a5061 +Author: Tim Brooks +Date: Wed Jan 23 12:05:39 2019 -0700 + + Add ability to listen to group of affix settings (#37679) + + Currently we have the ability to listen for setting changes to two group + affix settings. However, it is possible that we might have the need to + listen to more than two. This commit adds a method that allows consumer + to listen to a list of affix settings for changes. + +commit 0096f1b2e46d739f185725446b940787114f0310 +Author: Nhat Nguyen +Date: Wed Jan 23 13:41:13 2019 -0500 + + Ensure changes requests return the latest mapping version (#37633) + + Today we keep the mapping on the follower in sync with the leader's + using the mapping version from changes requests. There are two rare + cases where the mapping on the follower is not synced properly: + + 1. The returned mapping version (from ClusterService) is outdated than + the actual mapping. This happens because we expose the latest cluster + state in ClusterService after applying it to IndexService. + + 2. It's possible for the FollowTask to receive an outdated mapping than + the min_required_mapping. In that case, it should fetch the mapping + again; otherwise, the follower won't have the right mapping. + + Relates to #31140 + +commit d7fe4e57fe9dbe4a521ff2b9a2b6118d4450c50f +Author: Armin Braun +Date: Wed Jan 23 19:05:25 2019 +0100 + + Make Minio Setup more Reliable (#37747) + + * Retry starting Minio five times in case we run into a race between finding the free port and starting it up + * Closes #37680 + +commit 169cb387784653262c8de13666772a4942edbecf +Author: Jason Tedor +Date: Wed Jan 23 12:52:17 2019 -0500 + + Liberalize StreamOutput#writeStringList (#37768) + + In some cases we only have a string collection instead of a string list + that we want to serialize out. We have a convenience method for writing + a list of strings, but no such method for writing a collection of + strings. Yet, a list of strings is a collection of strings, so we can + simply liberalize StreamOutput#writeStringList to be more generous in + the collections that it accepts and write out collections of strings + too. On the other side, we do not have a convenience method for reading + a list of strings. This commit addresses both of these issues. + +commit 1c2ae9185cef956074b9daf1453cc3b4d84e2c23 +Author: Benjamin Trent +Date: Wed Jan 23 11:48:32 2019 -0600 + + Add PersistentTasksClusterService::unassignPersistentTask method (#37576) + + * Add PersistentTasksClusterService::unassignPersistentTask method + + * adding cancellation test + + * Adding integration test for unallocating tasks from a node + + * Addressing review comments + + * adressing minor PR comments + +commit e3672aa5516aa7f342c73743b49e6455b89e260f +Author: Igor Motov +Date: Wed Jan 23 12:25:54 2019 -0500 + + Tests: disable testRandomGeoCollectionQuery on tiny polygons (#37579) + + Due to https://issues.apache.org/jira/browse/LUCENE-8634 this test + may fail if a really tiny polygon is generated. This commit checks for + tiny polygons and skips the final check, which is expected to fail + until the lucene bug is fixed and new version of lucene is released. + +commit 427bc7f9401946b706e731dbb6e455fd99388abc +Author: Lee Hinman +Date: Wed Jan 23 10:18:08 2019 -0700 + + Use ILM for Watcher history deletion (#37443) + + * Use ILM for Watcher history deletion + + This commit adds an index lifecycle policy for the `.watch-history-*` indices. + This policy is automatically used for all new watch history indices. + + This does not yet remove the automatic cleanup that the monitoring plugin does + for the .watch-history indices, and it does not touch the + `xpack.watcher.history.cleaner_service.enabled` setting. + + Relates to #32041 + +commit f0fc6e800348e85748f82d078a89a123316ca27d +Author: Julie Tibshirani +Date: Wed Jan 23 08:51:05 2019 -0800 + + Make sure PutMappingRequest accepts content types other than JSON. (#37720) + +commit 647e2256984abd44d5d0820bc742a0af2061a2b8 +Author: Lee Hinman +Date: Wed Jan 23 09:46:31 2019 -0700 + + Retry ILM steps that fail due to SnapshotInProgressException (#37624) + + Some steps, such as steps that delete, close, or freeze an index, may fail due to a currently running snapshot of the index. In those cases, rather than move to the ERROR step, we should retry the step when the snapshot has completed. + + This change adds an abstract step (`AsyncRetryDuringSnapshotActionStep`) that certain steps (like the ones I mentioned above) can extend that will automatically handle a situation where a snapshot is taking place. When a `SnapshotInProgressException` is received by the listener wrapper, a `ClusterStateObserver` listener is registered to wait until the snapshot has completed, re-running the ILM action when no snapshot is occurring. + + This also adds integration tests for these scenarios (thanks to @talevy in #37552). + + Resolves #37541 + +commit d193ca8aaecbba7ae2ee7c62855d0e062742cc58 +Author: David Kyle +Date: Wed Jan 23 16:06:25 2019 +0000 + + Use disassociate in preference to deassociate (#37704) + +commit 2439f68745577c7b14192b8f3b0be0b5cbb84443 +Author: Armin Braun +Date: Wed Jan 23 16:39:02 2019 +0100 + + Delete Redundant RoutingServiceTests (#37750) + + * This test compleletly overrode the `reroute` method and hence did nothing put test the override itself + * Removed the test since it tests nothing and simplified `reroute` accordingly + +commit 6a9838359c1e33bacd034d84563b4f63ca174922 +Author: Nhat Nguyen +Date: Wed Jan 23 10:24:51 2019 -0500 + + Always return metadata version if metadata is requested (#37674) + + If the indices of a ClusterStateRequest are specified, we fail to + include the cluster state metadata version in the response. + + Relates #37633 + +commit 6a5d9d942a7537a17e7ef146c6fd0616dc3a815e +Author: David Roberts +Date: Wed Jan 23 13:50:31 2019 +0000 + + [TEST] Mute MlMappingsUpgradeIT testMappingsUpgrade + + Due to https://github.com/elastic/elasticsearch/issues/37763 + +commit 12f5b02fd0364de99b5d2d035cc7ffb579b43671 +Author: Luca Cavanna +Date: Wed Jan 23 13:53:37 2019 +0100 + + Streamline skip_unavailable handling (#37672) + + This commit moves the collectSearchShards method out of RemoteClusterService into TransportSearchAction that currently calls it. RemoteClusterService used to be used only for cross-cluster search but is now also used in cross-cluster replication where different API are called through the RemoteClusterAwareClient. There is no reason for the collectSearchShards and fetchShards methods to be respectively in RemoteClusterService and RemoteClusterConnection. The search shards API can be called through the RemoteClusterAwareClient too, the only missing bit is a way to handle failures based on the skip_unavailable setting for each cluster (currently only supported in RemoteClusterConnection#fetchShards) which is achieved by adding a isSkipUnavailable(String clusterAlias) method to RemoteClusterService. + This change is useful for #32125 as we will very soon need to also call the search API against remote clusters, which will be done through RemoteClusterAwareClient. In that case we will also need to support skip_unavailable when calling the search API so we need some way to handle the skip_unavailable setting like we currently do for the search_shards call. + + Relates to #32125 + +commit d5139e0590ed044c0950cfede5bac537a3fba2b3 +Author: Yannick Welsch +Date: Wed Jan 23 13:10:11 2019 +0100 + + Only bootstrap and elect node in current voting configuration (#37712) + + Adapts bootstrapping and leader election to only trigger on nodes that are actually part of the voting + configuration. + +commit 4ec3a6d922f302ed65b322506de8cdac3b8e1405 +Author: Simon Willnauer +Date: Wed Jan 23 12:38:44 2019 +0100 + + Ensure either success or failure path for SearchOperationListener is called (#37467) + + Today we have several implementations of executing SearchOperationListener + in SearchService. While all of them seem to be safe at least on, the one that + executes scroll searches can cause illegal execution of SearchOperationListener + that can then in-turn trigger assertions in ShardSearchStats. This change + adds a SearchOperationListenerExecutor that uses try-with blocks to ensure + listeners are called in a safe way. + + Relates to #37185 + +commit 100537fbc320d488c5b217999f01fdb63ce1e73e +Author: Daniel Mitterdorfer +Date: Wed Jan 23 12:19:54 2019 +0100 + + Target only specific index in update settings test + + With this commit we limit the update settings request to the index that + is used in `IndicesClientIT#testIndexPutSettings()`. This avoids + spurious exceptions involving other indices that might also be present + in the test cluster. + + Closes #36931 + Relates #37338 + +commit 1de286bfb3fc8190ef30ad4195f82e124b7417ff +Author: Daniel Mitterdorfer +Date: Wed Jan 23 12:15:00 2019 +0100 + + Add a note how to benchmark Elasticsearch + + With this commit we add a note that explains when to use benchmarks and + point the reader to the macrobenchmarking tool Rally for further + information. + + Relates #37694 + +commit f54a3b5f672c5f5728a43f0976ded7f6b7733fda +Author: Alpar Torok +Date: Wed Jan 23 12:09:40 2019 +0200 + + Don't use Groovy's `withDefault` (#37726) + + Closes #37061 + +commit 6130d151725c202fd11dfdbd267478edefc83f56 +Author: Tanguy Leroux +Date: Wed Jan 23 11:08:54 2019 +0100 + + Adapt SyncedFlushService (#37691) + +commit 701d89caa2e0e0d154b42a6afd50777a5213c3d3 +Author: Alexander Reelsen +Date: Wed Jan 23 11:00:37 2019 +0100 + + Mute FilterAggregatorTests#testRandom + + Relates #37743 + +commit daa2ec8a605d385a65b9ab3e89d016b3fd0dffe2 +Author: Alexander Reelsen +Date: Wed Jan 23 10:40:05 2019 +0100 + + Switch mapping/aggregations over to java time (#36363) + + This commit moves the aggregation and mapping code from joda time to + java time. This includes field mappers, root object mappers, aggregations with date + histograms, query builders and a lot of changes within tests. + + The cut-over to java time is a requirement so that we can support nanoseconds + properly in a future field mapper. + + Relates #27330 + +commit 7b3dd3022da261a6f299567091034548a80a5ebd +Author: David Roberts +Date: Wed Jan 23 09:37:37 2019 +0000 + + [ML] Update ML results mappings on process start (#37706) + + This change moves the update to the results index mappings + from the open job action to the code that starts the + autodetect process. + + When a rolling upgrade is performed we need to update the + mappings for already-open jobs that are reassigned from an + old version node to a new version node, but the open job + action is not called in this case. + + Closes #37607 + +commit b3f9becf5f99e718b751af81bb31ae1c0779fdcc +Author: Christoph Büscher +Date: Wed Jan 23 09:48:00 2019 +0100 + + Modify removal_of_types.asciidoc (#37648) + + After switching the default behaviour of "include_type_name" to "false" in 7.0, + some parts of the types removal documentation can be adapted as well. + +commit 6926a73d618be8bb8329e4f98eadcdf1fb4bb17d +Author: Christoph Büscher +Date: Wed Jan 23 09:46:49 2019 +0100 + + Fix edge case in PutMappingRequestTests (#37665) + + The recently introduced client side PutMappingRequestTests has an edge case that + leads to failing tests. When the "source" or the original request is `null` it + gets rendered to xContent as an empty object, which by the test class itself is + parsed back as an empty map. For this reason the original and the parsed request + differ (one has an empty source, one an empty map). This fixes this edge case by + assuming an empty map means a null source in the request. In practice the + distinction doesn't matter because all the client side request does is write + itself to xContent, which gives the same result regardless of whether `source` + is null or empty. + + Closes #37654 + +commit 95a6951f78524cf3ff64df5677f8a840b21fb141 +Author: Christoph Büscher +Date: Wed Jan 23 09:46:28 2019 +0100 + + Use new bulk API endpoint in the docs (#37698) + + This change switches to using the typeless bulk API endpoint in the + documentation snippets where possible + +commit 52ba407931093ba86538596639eaf35b2a858d01 +Author: Boaz Leskes +Date: Wed Jan 23 09:01:58 2019 +0100 + + Expose sequence number and primary terms in search responses (#37639) + + Users may require the sequence number and primary terms to perform optimistic concurrency control operations. Currently, you can get the sequence number via the `docvalues_fields` API but the primary term is not accessible because it is maintained by the `SeqNoFieldMapper` and the infrastructure can't find it. + + This commit adds a dedicated sub fetch phase to return both numbers that is connected to a new `seq_no_primary_term` parameter. + +commit 534ba1dd349a4baadd6ab8dc63dab0a849733804 +Author: Andrey Ershov +Date: Wed Jan 23 07:23:06 2019 +0100 + + Remove LicenseServiceClusterNotRecoveredTests (#37528) + + While tests migration from Zen1 to Zen2, we've encountered this test. + This test is organized as follows: + + Starts the first cluster node. + Starts the second cluster node. + Checks that license is active. + Interesting fact that adding assertLicenseActive(true) between 1 + and 2 also makes the test pass. + assertLicenseActive retrieves XPackLicenseState from the nodes + and checks that active flag is set. It's set to true even before + the cluster is initialized. + + So this test does not make sense. + +commit 7c6566e14c22d80db7e91a422c5b3c0df7b10b2d +Author: Andrey Ershov +Date: Wed Jan 23 07:22:41 2019 +0100 + + Migrate SpecificMasterNodesIT to Zen2 (#37532) + + 1. testSimpleOnlyMasterNodeElection - requires cluster bootstrap when + the first master node is started. + 2. testElectOnlyBetweenMasterNodes - requires cluster bootstrap when + the first master node is started and requires adding voting exclusion + before shutting down the first master node. + 3. testAliasFilterValidation - requires cluster bootstrap when the + first master node is started. + +commit e2e00cd2450058384ce4bbdf16b130824a75f81a +Author: Andrey Ershov +Date: Wed Jan 23 07:21:26 2019 +0100 + + Fix MetaStateFormat tests + + + It's not safe to continue writing state using MetaDataStateFormat + after dirty WriteStateException occurred if it's not recovered by + successful subsequent state write. + + We've encountered test failure of testFailRandomlyAndReadAnyState. + The test breaks in the following way. There are 3 state paths. And what + happens next + + Successful write at the beginning of the test yields 0 0 0 state + files in the directories. + 1st write in the loop is unsuccessful, but not dirty - 0 0 0. + 2nd write in the loop is not successful and dirty (failure during + fsync), however before removing new files we have 1 1 1. But now during + deletion, the first deletion fails and we get - 1 0 0. + 3rd write in the loop is unsuccessful, but not dirty - so we want to + keep old generation, which happens to be the 1st generation, so now we + have 1 x x in state folders. Now we assert that we either load 0 or 1 + state from the state folders and select only 2rd and 3th folder to + emulate disk failures - this results in NPE because there is nothing in + these folders. + Fortunately, this won’t be a problem in real life, because if there is + a dirty exception, we shut down the node and make sure we perform a + successful write on the node startup. + +commit 942fc13af5985eda8723f9c072194c7e4c0dc5fc +Author: Mayya Sharipova +Date: Tue Jan 22 16:49:03 2019 -0500 + + Use plain text instead of latexmath + + As latexmath is not rendered, using plain text instead + + Closes #37718 + +commit c28479819efa201ab141a8c5c5ccd856c5a00116 +Author: olcbean +Date: Tue Jan 22 21:36:42 2019 +0100 + + Fix a typo in a warning message in TestFixturesPlugin (#37631) + +commit 992bfd2064a4074b8eb278b6354ae68e9fa87fda +Author: Julie Tibshirani +Date: Tue Jan 22 12:13:01 2019 -0800 + + Remove additional references to 'type' from the _bulk documentation. (#37722) + +commit 940f6ba4c1192e1663168ebce4a42da6f0352d87 +Author: Brandon Kobel +Date: Tue Jan 22 12:09:08 2019 -0800 + + Remove kibana_user and kibana_dashboard_only_user index privileges (#37441) + + * Remove kibana_user and kibana_dashboard_only_user .kibana* index privileges + + * Removing unused imports + +commit eb43ab6d6081ca4f239a0e4c2672658b1f9d9a26 +Author: Tim Brooks +Date: Tue Jan 22 10:57:37 2019 -0700 + + Implement leader rate limiting for file restore (#37677) + + This is related to #35975. This commit implements rate limiting on the + leader side using the CombinedRateLimiter. + +commit 2ba9e361aba7df6e9c0b5fa8fae3d219d50ddf09 +Author: Zachary Tong +Date: Tue Jan 22 12:38:55 2019 -0500 + + Add helper classes to determine if aggs have a value (#36020) + + This adds a set of helper classes to determine if an agg "has a value". + This is needed because InternalAggs represent "empty" in different + manners according to convention. Some use `NaN`, `+/- Inf`, `0.0`, etc. + + A user can pass the Internal agg type to one of these helper methods + and it will report if the agg contains a value or not, which allows the + user to differentiate "empty" from a real `NaN`. + + These helpers are best-effort in some cases. For example, several + pipeline aggs share a single return class but use different conventions + to mark "empty", so the helper uses the loosest definition that applies + to all the aggs that use the class. + + Sums in particular are unreliable. The InternalSum simply returns 0.0 + if the agg is empty (which is correct, no values == sum of zero). But this + also means the helper cannot differentiate from "empty" and `+1 + -1`. + +commit 715719ee3b5ee29edd008571d1ce3bf975846a25 +Author: Jason Tedor +Date: Tue Jan 22 12:29:24 2019 -0500 + + Remove warn-date from warning headers (#37622) + + This commit removes the warn-date from warning headers. Previously we + were stamping every warning header with when the request + occurred. However, this has a severe performance penalty when + deprecation logging is called frequently, as obtaining the current time + and formatting it properly is expensive. A previous change moved to + using the startup time as the time to stamp on every warning header, but + this was only to prove that the timestamping was expensive. Since the + warn-date is optional, we elect to remove it from the warning + header. Prior to this commit, we worked in Kibana to make the warn-date + treated as optional there so that we can follow-up in Elasticsearch and + remove the warn-date. This commit does that. + +commit 256e01ca92c43f3ccdc87bfc95031cbcda82e04a +Author: Christoph Büscher +Date: Tue Jan 22 17:34:13 2019 +0100 + + Fix potential NPE in UsersTool (#37660) + + It looks like the output of FileUserPasswdStore.parseFile shouldn't be wrapped + into another map since its output can be null. Doing this wrapping after the null + check (which potentially raises an exception) instead. + +commit 5c1a1f7ac1740fc902cc77ad47e7a2441b5536b4 +Author: Ioannis Kakavas +Date: Tue Jan 22 17:26:36 2019 +0200 + + Use PEM files for PkiOptionalClientAuthTests (#37683) + + Use PEM files for the key/cert for TLS on the http layer of the + node instead of a JKS keystore so that the tests can also run + in a FIPS 140 JVM . + + Resolves: #37682 + +commit 3f2723366e4c63da63e400a60211a6c7550ebf1b +Author: Alpar Torok +Date: Tue Jan 22 17:16:03 2019 +0200 + + Mute failing test + + Tracking #37708 + +commit 34f2d2ec916909e107a85c26b1cccdda3026aac9 +Author: Christoph Büscher +Date: Tue Jan 22 15:13:52 2019 +0100 + + Remove remaining occurances of "include_type_name=true" in docs (#37646) + +commit 7507af29fa0ae8569e3551d7815fc55afe5af014 +Author: Andrei Stefan +Date: Tue Jan 22 14:55:28 2019 +0200 + + SQL: Return Intervals in SQL format for CLI (#37602) + + * Add separate CLI Mode + * Use the correct Mode for cursor close requests + * Renamed CliFormatter and have different formatting behavior for CLI and "text" format. + +commit 23ba9008409444a0f1a48f6fd73b545307efe192 +Author: Yannick Welsch +Date: Tue Jan 22 13:53:10 2019 +0100 + + Publish to masters first (#37673) + + Prefer publishing to master-eligible nodes first, so that cluster state updates are committed more + quickly, and master-eligible nodes also turned more quickly into followers after a leader election. + +commit 3fad1eeaed45761d734ece7a1c717dc6c6e6ccd2 +Author: David Kyle +Date: Tue Jan 22 12:44:45 2019 +0000 + + Un-assign persistent tasks as nodes exit the cluster (#37656) + + PersistentTasksClusterService decides if a task should be reassigned by + checking there is a node in the cluster with the same Id. If a node is + restarted PersistentTasksClusterService may not observe the change and + decide the task still has a valid assignment because the node's + ephemeral Id is not used in that decision. This change un-assigns tasks + as the nodes in the cluster change. + +commit 228611843c6d165087c2d957df57b94b9d93a143 +Author: Henning Andersen <33268011+henningandersen@users.noreply.github.com> +Date: Tue Jan 22 13:27:12 2019 +0100 + + Fail start of non-data node if node has data (#37347) + + * Fail start of non-data node if node has data + + Check that nodes started with node.data=false cannot start if they have + shard data to avoid (old) indexes being resurrected into the cluster in red status. + + Issue #27073 + +commit 2a7b7ccf1c6c1503a2a1c5aa5a13dfa9b122625b +Author: Yannick Welsch +Date: Tue Jan 22 12:51:03 2019 +0100 + + Use cancel instead of timeout for aborting publications (#37670) + + When publications were cancelled because a node turned to follower or candidate, it would still + show as time out, which can be confusing in the logs. This change adapts the improper call of + onTimeout by generalizing it to a cancel method. + +commit ef2f5e4a136fff487e3c6f30656d3e222d74ac25 +Author: Martijn van Groningen +Date: Tue Jan 22 12:48:05 2019 +0100 + + Follow stats api should return a 404 when requesting stats for a non existing index (#37220) + + Currently it returns an empty response with a 200 response code. + + Closes #37021 + +commit 0a93a0358bfed106bf397a24c0590008225ad21f +Author: Christoph Büscher +Date: Tue Jan 22 12:12:21 2019 +0100 + + Remove deprecated FieldNamesFieldMapper.Builder#index (#37305) + + The method calls "enabled" in addition to what the super.index() does, but this + seems to be done explicitely now in the TypeParsers `parse` method. The removed + method has been deprecated since at least 6.0. Also making some of the Builders + methods and ctos private since they are only used internally in this class. + +commit 757932a9751757c9c3689032fc4190dbfbe831a6 +Author: Daniel Mitterdorfer +Date: Tue Jan 22 12:04:29 2019 +0100 + + Document that date math is locale independent + + With this commit we add a note to the API conventions documentation that + all date math expressions are resolved independently of any locale. This + behavior might be puzzling to users that try to specify a different + calendar than a Gregorian calendar. + + Closes #37330 + Relates #37663 + +commit 5db7ed22a0aab6e630a89952ec94327d205fc3a3 +Author: David Turner +Date: Tue Jan 22 11:03:51 2019 +0000 + + Bootstrap a Zen2 cluster once quorum is discovered (#37463) + + Today when bootstrapping a Zen2 cluster we wait for every node in the + `initial_master_nodes` setting to be discovered, so that we can map the + node names or addresses in the `initial_master_nodes` list to their IDs for + inclusion in the initial voting configuration. This means that if any of + the expected master-eligible nodes fails to start then bootstrapping will + not occur and the cluster will not form. This is not ideal, and we would + prefer the cluster to bootstrap even if some of the master-eligible nodes + do not start. + + Safe bootstrapping requires that all pairs of quorums of all initial + configurations overlap, and this is particularly troublesome to ensure + given that nodes may be concurrently and independently attempting to + bootstrap the cluster. The solution is to bootstrap using an initial + configuration whose size matches the size of the expected set of + master-eligible nodes, but with the unknown IDs replaced by "placeholder" + IDs that can never belong to any node. Any quorum of received votes in any + of these placeholder-laden initial configurations is also a quorum of the + "true" initial set of master-eligible nodes, giving the guarantee that it + intersects all other quorums as required. + + Note that this change means that the initial configuration is not + necessarily robust to any node failures. Normally the cluster will form and + then auto-reconfigure to a more robust configuration in which the + placeholder IDs are replaced by the IDs of genuine nodes as they join the + cluster; however if a node fails between bootstrapping and this + auto-reconfiguration then the cluster may become unavailable. This we feel + to be less likely than a node failing to start at all. + + This commit also enormously simplifies the cluster bootstrapping process. + Today, the cluster bootstrapping process involves two (local) transport actions + in order to support a flexible bootstrapping API and to make it easily + accessible to plugins. However this flexibility is not required for the current + design so it is adding a good deal of unnecessary complexity. Here we remove + this complexity in favour of a much simpler ClusterBootstrapService + implementation that does all the work itself. + +commit e9fcb25a2876594cbadbc6f229e524e3dde8875d +Author: Adrien Grand +Date: Tue Jan 22 11:44:29 2019 +0100 + + Upgrade to lucene-8.0.0-snapshot-83f9835. (#37668) + + This snapshot uses a new file format for doc-values which is expected to make + advance/advanceExact perform faster on sparse fields: + https://issues.apache.org/jira/browse/LUCENE-8585 + +commit 74d1cfbf7e748868fed35bd2fe55f2788cded7b8 +Author: Alpar Torok +Date: Tue Jan 22 10:50:05 2019 +0200 + + Mute failing test + + Tracking ##37687 + +commit 4fb68ea195c010cc3fa381d849291ace28117980 +Author: Alexander Reelsen +Date: Tue Jan 22 09:42:17 2019 +0100 + + Fix java time formatters that round up (#37604) + + In order to be able to parse epoch seconds and epoch milli seconds own + java time fields had been introduced. These fields are however not + compatible with the way that java time allows one to configure default + fields (when a part of a timestamp cannot be read then a default value + is added), which is used for the formatters that are rounding up to the + next value. + + This commit allows java date formatters to configure its round up parsing + by setting default values via a consumer. By default all formats are setting + JavaDateFormatter.ROUND_UP_BASE_FIELDS for rounding up. The epoch + however parsers both need to set different fields. The merged date + formatters do not set any fields, they just append all the round up formatters. + + Also the formatter now properly copies the locale and the timezone, + fractional parsing has been set to nano seconds with proper width. + +commit 3e1e1b0b37cf4082a1628704cd3cd303445e38da +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jan 22 19:35:17 2019 +1100 + + Removes awaits fix as the fix is in. (#37676) + + The PR for the fix has been merged. + https://github.com/elastic/elasticsearch/pull/37661 + but the awaits fix annotation was not removed. + +commit 17d704347e171abd9e2b190563eee9776d715608 +Author: Alpar Torok +Date: Tue Jan 22 10:27:55 2019 +0200 + + Mute failing test + + Tracking #37685 + +commit 0290547ad7455d9d2330665eb7520b9908e0860b +Author: Tanguy Leroux +Date: Tue Jan 22 09:22:33 2019 +0100 + + Ensure that max seq # is equal to the global checkpoint when creating ReadOnlyEngines (#37426) + + Since version 6.7.0 the Close Index API guarantees that all translog + operations have been correctly flushed before the index is closed. If + the index is reopened as a Frozen index (which uses a ReadOnlyEngine) + we can verify that the maximum sequence number from the last Lucene + commit is indeed equal to the last known global checkpoint and refuses + to open the read only engine if it's not the case. In this PR the check is + only done for indices created on or after 6.7.0 as they are guaranteed + to be closed using the new Close Index API. + + Related #33888 + +commit a713183cab8adfb7c8bc81e0e949c73d7d404f37 +Author: Alpar Torok +Date: Tue Jan 22 10:04:12 2019 +0200 + + Mute failing discovery disruption tests + + Tracking #37539 + +commit 257f3eff2283a691802da9042f4b46934185ad7a +Author: Tomas Della Vedova +Date: Tue Jan 22 09:07:48 2019 +0100 + + Add note about how the body is referenced (#33935) + +commit 90ae556d97dc05f9a5a34b8bd08234acfd5ffe28 +Author: Andrei Stefan +Date: Tue Jan 22 10:01:51 2019 +0200 + + Define constants for REST requests endpoints in tests (#37610) + +commit 7394892b4c36f44d5addb2abd2dbca7453cb2d79 +Author: Nhat Nguyen +Date: Mon Jan 21 21:35:10 2019 -0500 + + Make prepare engine step of recovery source non-blocking (#37573) + + Relates #37174 + +commit ca4b5861c8c84a8c1415a3dda89f83720c3b6da8 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jan 22 09:34:40 2019 +1100 + + Fix a test failure in CompositeRolesStoreTests (#37661) + + Due to missing stubbing for `NativePrivilegeStore#getPrivileges` + the test `testNegativeLookupsAreCached` failed + when the superuser role name was present in the role names. + This commit adds missing stubbing. + + Closes: #37657 + +commit 21838d73b5f26490967d0db3d6667a701de6387a +Author: Tim Brooks +Date: Mon Jan 21 14:14:18 2019 -0700 + + Extract message serialization from `TcpTransport` (#37034) + + This commit introduces a NetworkMessage class. This class has two + subclasses - InboundMessage and OutboundMessage. These messages can + be serialized and deserialized independent of the transport. This allows + more granular testing. Additionally, the serialization mechanism is now + a simple Supplier. This builds the framework to eventually move the + serialization of transport messages to the network thread. This is the + one serialization component that is not currently performed on the + network thread (transport deserialization and http serialization and + deserialization are all on the network thread). + +commit f516d68fb28ef56271f2784715a68908570d9b87 +Author: Tim Brooks +Date: Mon Jan 21 13:50:56 2019 -0700 + + Share `NioGroup` between http and transport impls (#37396) + + Currently we create dedicated network threads for both the http and + transport implementations. Since these these threads should never + perform blocking operations, these threads could be shared. This commit + modifies the nio-transport to have 0 http workers be default. If the + default configs are used, this will cause the http transport to be run + on the transport worker threads. The http worker setting will still exist + in case the user would like to configure dedicated workers. Additionally, + this commmit deletes dedicated acceptor threads. We have never had these + for the netty transport and they can be added back if a need is + determined in the future. + +commit 9a34b20233b7b780dc14c9781c998d7f34f2333e +Author: Ryan Ernst +Date: Mon Jan 21 12:37:17 2019 -0800 + + Simplify integ test distribution types (#37618) + + The integ tests currently use the raw zip project name as the + distribution type. This commit simplifies this specification to be + "default" or "oss". Whether zip or tar is used should be an internal + implementation detail of the integ test setup, which can (in the future) + be platform specific. + +commit 0d7831ca6a5b2e3f5c3a9c835f10a44c290fa4d7 +Author: Albert Zaharovits +Date: Mon Jan 21 19:02:21 2019 +0200 + + Checkstyle PutRoleRequest + +commit f349372fbafde7dedfbe03064e0fa7f53933453e +Author: Albert Zaharovits +Date: Mon Jan 21 18:39:53 2019 +0200 + + Mute test. + + Relates #37657 + +commit 5843aba8bdcf367d158570e1c96f8fa291ad1c37 +Author: Albert Zaharovits +Date: Mon Jan 21 18:35:44 2019 +0200 + + Checkstyle PutRoleRequestTests + +commit 2c02b298d35311e0773120b9fd627ece2145dd18 +Author: Albert Zaharovits +Date: Mon Jan 21 18:14:02 2019 +0200 + + Fix PutRoleRequestTests + + Closes #37662 + +commit 24c5dd498fcb8a6798359da86fde4935ae06650a +Author: Alexander Reelsen +Date: Mon Jan 21 16:11:42 2019 +0100 + + Mute PutRoleRequestTests.testSerializationBetweenV63AndV70 + + Relates #37662 + +commit 0ae7f8630cd9e4c45824658d35e80b32d384a66b +Author: David Kyle +Date: Mon Jan 21 14:12:20 2019 +0000 + + Document ml datafeed Id limitations (#37653) + +commit 7173aa213ab219bbac37de417cf6fe1246592467 +Author: Alexander Reelsen +Date: Mon Jan 21 14:02:53 2019 +0100 + + Mute PutMappingRequestTests.testSerialization + + This test failed in a couple of branches/PRs, thus muting for now. + + Relates #37654 + +commit a3180fd136f01e94299872e89b88a38ec334c089 +Author: Martijn van Groningen +Date: Mon Jan 21 13:58:16 2019 +0100 + + Decrease ccr.auto_follow.wait_for_metadata_timeout in ccr rolling upgrade test, + in order to reduce the likelihood the test fails because of timeing issues. + + Relates #37231 + +commit 3a3f5b39c38cd053cd6dcdebea8141dd2f31120e +Author: Armin Braun +Date: Mon Jan 21 13:10:33 2019 +0100 + + Fix Race in Concurrent Snapshot Delete and Create (#37612) + + * The repo id was determined wrong when the delete picked up on an in progress snapshot + * NOTE: This solution is still a best-effort fix and there's a slight chance of running into concurrency issues here + when multiple create and delete requests for the same snapshot name are happening concurrently, but these require a sequence + of multiple cluster state updates between the changed method reading the genId and submitting its cluster state update task + * Added test reproduced the issue reliably in about 50% of runs + * Closes #37581 + +commit f70ec3badbcbdf0d913bfbba31660ca70148c6a4 +Author: Albert Zaharovits +Date: Mon Jan 21 14:07:19 2019 +0200 + + Fix PutRoleRequestTests + + Related ff0f5402 + +commit 0631322dda60d4d61f1bc7ae33ac379fb4e2787a +Author: Albert Zaharovits +Date: Mon Jan 21 13:45:46 2019 +0200 + + Stream version nit after ff0f540 and ce60585 + +commit 468bae29f7df81df5d44f1e8324468584be9b3e6 +Author: markharwood +Date: Mon Jan 21 11:52:21 2019 +0000 + + Mute test + Tracking #37652 + +commit 3a2b6165799966b5a0c5de0984ffad27f8f41b57 +Author: Alpar Torok +Date: Mon Jan 21 12:52:25 2019 +0200 + + Use zen2 with testclusters (#37352) + +commit 09a6ba50efef1f9112665d7f1ee5e5faa4db6ff5 +Author: Luca Cavanna +Date: Mon Jan 21 11:51:47 2019 +0100 + + Add support for merging multiple search responses into one (#37566) + + This will be used in cross-cluster search when reduction will be + performed locally on each cluster. The CCS coordinating node will send + one search request per remote cluster involved and will get one search + response back from each one of them. Such responses contain all the info + to be able to perform an additional reduction and return results back + to the user. + + Relates to #32125 + +commit 14d74eb30b11f0534925f1b2b9b2098c3421b69e +Author: Alpar Torok +Date: Mon Jan 21 11:13:15 2019 +0200 + + Mute test on windows + + Tracking #37342 + +commit c9c088ff0212ba144b64326c4d297183b1c0eb8f +Author: Ryan Ernst +Date: Sun Jan 20 23:33:11 2019 -0800 + + Use ivy pattern for elasticsearch artifacts download (#37557) + + The rpm, deb and tar distributions were removed some time ago from maven + central. The zip distribution still exists there, but it does not need + to. Instead, this commit sets up an ivy repository with pattern pointing + to the elasticsearch artifacts download service. Note that the + integ-test-zip remains in maven central, since it is not present in the + download service. + +commit 88f4b0a326ffa82dfdb10a8f04d3d17557938278 +Author: Martijn van Groningen +Date: Mon Jan 21 07:54:51 2019 +0100 + + Do not set fatal exception when shard follow task is stopped. (#37603) + + When shard follow task is cancelled while fetching operations then + the fatal exception field should not be set. + +commit 614950ef1c2afc35ee37e1ec7c0e5189ec9b5b2f +Author: Ryan Ernst +Date: Sun Jan 20 22:47:43 2019 -0800 + + Handle requiring versioned java home during execution (#37599) + + Currently bwc builds require different java home environment variables + depending on the version of elasticsearch being built. The java home + version checks are run at the end of gradle configuration, when the task + graph is ready. However, we do not know which versions are needed for + bwc builds until execution time, when we have finished checking out the + version of elasticsearch to be built. This commit accounts for late java + home checks, checking immediately instead of delaying the check. + + closes #37586 + +commit ff0f5402558b986756ee3a7ccceb7dd6cc514620 +Author: Albert Zaharovits +Date: Sun Jan 20 23:19:40 2019 +0200 + + Permission for restricted indices (#37577) + + This grants the capability to grant privileges over certain restricted + indices (.security and .security-6 at the moment). + It also removes the special status of the superuser role. + + IndicesPermission.Group is extended by adding the `allow_restricted_indices` + boolean flag. By default the flag is false. When it is toggled, you acknowledge + that the indices under the scope of the permission group can cover the + restricted indices as well. Otherwise, by default, restricted indices are ignored + when granting privileges, thus rendering them hidden for authorization purposes. + This effectively adds a confirmation "check-box" for roles that might grant + privileges to restricted indices. + + The "special status" of the superuser role has been removed and coded as + any other role: + ``` + new RoleDescriptor("superuser", + new String[] { "all" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices("*") + .privileges("all") + .allowRestrictedIndices(true) + // this ----^ + .build() }, + new RoleDescriptor.ApplicationResourcePrivileges[] { + RoleDescriptor.ApplicationResourcePrivileges.builder() + .application("*") + .privileges("*") + .resources("*") + .build() + }, + null, new String[] { "*" }, + MetadataUtils.DEFAULT_RESERVED_METADATA, + Collections.emptyMap()); + ``` + In the context of the Backup .security work, this allows the creation of a + "curator role" that would permit listing (get settings) for all indices + (including the restricted ones). That way the curator role would be able to + ist and snapshot all indices, but not read or restore any of them. + + Supersedes #36765 + Relates #34454 + +commit 5308746270292775ddb14c377e96871b0e8e256e +Author: Albert Zaharovits +Date: Sun Jan 20 12:51:24 2019 +0200 + + Remove Watcher Account "unsecure" settings (#36736) + + Removes all sensitive settings (passwords, auth tokens, urls, etc...) for + watcher notifications accounts. These settings were deprecated (and + herein removed) in favor of their secure sibling that is set inside the + elasticsearch keystore. For example: + `xpack.notification.email.account..smtp.password` + is no longer a valid setting, and it is replaced by + `xpack.notification.email.account..smtp.secure_password` + +commit fc99eb3e65b7a8fb25517782c760f7a1692bf2cc +Author: Ryan Ernst +Date: Sat Jan 19 16:16:58 2019 -0800 + + Add cache cleaning task for ML snapshot (#37505) + + The ML subproject of xpack has a cache for the cpp artifact snapshots + which is checked on each build. The cache is outside of the build dir so + that it is not wiped on a typical clean, as the artifacts can be large + and do not change often. This commit adds a cleanCache task which will + wipe the cache dir, as over time the size of the directory can become + bloated. + +commit 9b32f57cf1a3103c1e845840119adc7f4b2bc29d +Author: Michael Basnight +Date: Fri Jan 18 19:01:35 2019 -0600 + + Update jdk used by the docker builds (#37621) + + With the release of 11.0.2, the old URLs no longer work. This exposed a + few small bugs in the gradle config. One was that --no-cache was not + present in the docker build command, so it was not failing at + first. Then once only the ext.expansions was changed and the docker + build task was not, it was not executing it. + +commit b4c18a9eb41f64aa21fa97fff4d92aad37e617a5 +Author: Julie Tibshirani +Date: Fri Jan 18 14:43:50 2019 -0800 + + Remove an unused constant in PutMappingRequest. + +commit c03308a071a4e7b6f740d93719eeba959dc1f0d7 +Author: Michael Basnight +Date: Fri Jan 18 17:50:51 2019 -0600 + + Update get users to allow unknown fields (#37593) + + The subparser in get users allows for unknown fields. This commit sets + the value to true for the parser and modifies the test such that it + accurately tests it. + + Relates #36938 + +commit fe753ee1d23aa104ae37907a064eb2a6dfa5860b +Author: Tim Brooks +Date: Fri Jan 18 16:31:21 2019 -0700 + + Do not add index event listener if CCR disabled (#37432) + + Currently we add the CcrRestoreSourceService as a index event + listener. However, if ccr is disabled, this service is null and we + attempt to add a null listener throwing an exception. This commit only + adds the listener if ccr is enabled. + +commit cd412893965e244cc708dae25f8d5bf44baac723 +Author: Tim Brooks +Date: Fri Jan 18 14:48:20 2019 -0700 + + Add local session timeouts to leader node (#37438) + + This is related to #35975. This commit adds timeout functionality to + the local session on a leader node. When a session is started, a timeout + is scheduled using a repeatable runnable. If the session is not accessed + in between two runs the session is closed. When the sssion is closed, + the repeating task is cancelled. + + Additionally, this commit moves session uuid generation to the leader + cluster. And renames the PutCcrRestoreSessionRequest to + StartCcrRestoreSessionRequest to reflect that change. + +commit adae233f77238a12911733d15c45e5acbe689512 +Author: Jason Tedor +Date: Fri Jan 18 16:42:25 2019 -0500 + + Add some deprecation optimizations (#37597) + + This commit optimizes some of the performance issues from using + deprecation logging: + - we optimize encoding the deprecation value + - we optimize formatting the deprecation string + - we optimize away getting the current time (by using cached startup + time) + +commit 106f900dfb3ac49ab45ec20a7dae43753de2e4c8 +Author: Tal Levy +Date: Fri Jan 18 13:40:00 2019 -0800 + + refactor inner geogrid classes to own class files (#37596) + + To make further refactoring of GeoGrid aggregations + easier (related: #30320), splitting out these inner + class dependencies into their own files makes it + easier to map the relationship between classes + +commit 88b981056712bb2d6c668c365e357c605e6ffecd +Author: Gordon Brown +Date: Fri Jan 18 14:24:34 2019 -0700 + + Remove obsolete deprecation checks (#37510) + + * Remove obsolete deprecation checks + + This also updates the old-indices check to be appropriate for the 7.x + series of releases, and leaves it as the only deprecation check in + place. + + * Add toString to DeprecationIssue + + * Bring filterChecks across from 6.x + + * License headers + +commit 12cdf1cba44d84fd8a8aa3cae39e76a8b359b595 +Author: Benjamin Trent +Date: Fri Jan 18 15:08:53 2019 -0600 + + ML: Add support for single bucket aggs in Datafeeds (#37544) + + Single bucket aggs are now supported in datafeed aggregation configurations. + +commit 5384162a42b066ee4bcadba6ab6d699f6966747f +Author: Benjamin Trent +Date: Fri Jan 18 14:32:34 2019 -0600 + + ML: creating ML State write alias and pointing writes there (#37483) + + * ML: creating ML State write alias and pointing writes there + + * Moving alias check to openJob method + + * adjusting concrete index lookup for ml-state + +commit 8da7a27f3b12c307bcbfaff7fdfa9d2a1f369b6a +Author: Julie Tibshirani +Date: Fri Jan 18 12:28:31 2019 -0800 + + Deprecate types in the put mapping API. (#37280) + + From #29453 and #37285, the `include_type_name` parameter was already present and defaulted to false. This PR makes the following updates: + - Add deprecation warnings to `RestPutMappingAction`, plus tests in `RestPutMappingActionTests`. + - Add a typeless 'put mappings' method to the Java HLRC, and deprecate the old typed version. To do this cleanly, I opted to create a new `PutMappingRequest` object that differs from the existing server one. + +commit a3030c51e272bff2e62244c1bdce5f93afd1932f +Author: Martijn van Groningen +Date: Fri Jan 18 21:05:03 2019 +0100 + + [ILM] Add unfollow action (#36970) + + This change adds the unfollow action for CCR follower indices. + + This is needed for the shrink action in case an index is a follower index. + This will give the follower index the opportunity to fully catch up with + the leader index, pause index following and unfollow the leader index. + After this the shrink action can safely perform the ilm shrink. + + The unfollow action needs to be added to the hot phase and acts as + barrier for going to the next phase (warm or delete phases), so that + follower indices are being unfollowed properly before indices are expected + to go in read-only mode. This allows the force merge action to execute + its steps safely. + + The unfollow action has three steps: + * `wait-for-indexing-complete` step: waits for the index in question + to get the `index.lifecycle.indexing_complete` setting be set to `true` + * `wait-for-follow-shard-tasks` step: waits for all the shard follow tasks + for the index being handled to report that the leader shard global checkpoint + is equal to the follower shard global checkpoint. + * `pause-follower-index` step: Pauses index following, necessary to unfollow + * `close-follower-index` step: Closes the index, necessary to unfollow + * `unfollow-follower-index` step: Actually unfollows the index using + the CCR Unfollow API + * `open-follower-index` step: Reopens the index now that it is a normal index + * `wait-for-yellow` step: Waits for primary shards to be allocated after + reopening the index to ensure the index is ready for the next step + + In the case of the last two steps, if the index in being handled is + a regular index then the steps acts as a no-op. + + Relates to #34648 + + Co-authored-by: Martijn van Groningen + Co-authored-by: Gordon Brown + +commit a2bdfb9041528e52ccc21369886f811802d12ce6 +Author: Ryan Ernst +Date: Fri Jan 18 11:49:40 2019 -0800 + + Packaging: Update marker used to allow ELASTIC_PASSWORD (#37243) + + This commit updates the file docker's entrypoint script looks for when + deciding to process the ELASTIC_PASSWORD env var. The x-pack subdir + of bin no longer exists in 7.0, where the backcompat layer for x-pack + script locations was removed. + + closes #37240 + +commit 642e45e9e670b654042088ca4ed80a287cc57c8f +Author: jaymode +Date: Fri Jan 18 12:24:11 2019 -0700 + + Fix setting openldap realm ssl config + + This change fixes the setup of the SSL configuration for the test + openldap realm. The configuration was missing the realm identifier so + the SSL settings being used were just the default JDK ones that do not + trust the certificate of the idp fixture. + + See #37591 + +commit fc1c47649ad000b8ceb02a5c7d9d85b7590df8f0 +Author: Heather McCartney +Date: Fri Jan 18 19:16:50 2019 +0000 + + Document the need for JAVA11_HOME (#37589) + + This commit updates the contribution docs to include java11 as a requirement for building and testing Elasticsearch. + +commit 54af8a4e7a51d5a078e07a5401a8bab4fa040687 +Author: Igor Motov +Date: Fri Jan 18 14:03:48 2019 -0500 + + SQL: fix object extraction from sources (#37502) + + Throws an exception if hit extractor tries to retrieve unsupported + object. For example, selecting "a" from `{"a": {"b": "c"}}` now throws + an exception instead of returning null. + + Relates to #37364 + +commit 633bd09be055f89a42a89c132c2c4480978a035a +Author: Albert Zaharovits +Date: Fri Jan 18 18:53:58 2019 +0200 + + Nit in settings.gradle for Eclipse + + Fixes ``./gradlew eclipse` failure introduced in 6d99e790b3ee + +commit de55b4dfd1fe07d5b027a3a61e88a26796862153 +Author: Jack Conradson +Date: Fri Jan 18 09:13:49 2019 -0800 + + Add types deprecation to script contexts (#37554) + + This adds deprecation to _type in the script contexts for ingest and update. + This adds a DeprecationMap that wraps the ctx Map containing _type for these + specific contexts. + +commit 6846666b6bb4a10db7c6f653ba6f45ce5b0c283c +Author: Martijn van Groningen +Date: Fri Jan 18 16:37:21 2019 +0100 + + Add ccr follow info api (#37408) + + * Add ccr follow info api + + This api returns all follower indices and per follower index + the provided parameters at put follow / resume follow time and + whether index following is paused or active. + + Closes #37127 + + * iter + + * [DOCS] Edits the get follower info API + + * [DOCS] Fixes link to remote cluster + + * [DOCS] Clarifies descriptions for configured parameters + +commit 377d96e376a9527c70804a4e7fbf5918b97d41ac +Author: Yannick Welsch +Date: Fri Jan 18 16:36:42 2019 +0100 + + Remove initial_master_nodes on node restart (#37580) + + Some tests (e.g. testRestoreIndexWithShardsMissingInLocalGateway) were split-braining since + being switched to Zen2 because the bootstrap setting was left around when nodes got restarted + with data folders wiped. + + The test in question here was starting one node (which autobootstrapped to that single node), then + another node. The first node was then shut down (after excluding it from the voting configuration), + its data folder wiped, and restarted. After restart, the node had an empty data folder yet + initial_master_nodes set to itself (i.e. same name). This made the node sometimes form a cluster of + its own, and not rejoin the existing cluster with the other node. + +commit 604422c6c51b5095e6b1d463d31916441661fb87 +Author: Michael Basnight +Date: Fri Jan 18 09:20:32 2019 -0600 + + Update Execute Watch to allow unknown fields (#37498) + + ExecuteWatchResponse did not allow unknown fields. This commit fixes the + test and ConstructingObjectParser such that it does now allow unknown + fields. It also creates a new client side test for the response. + + Relates #36938 + +commit 7597b7ce2bd280401fcbfbeb281dfbb205830d75 +Author: Ioannis Kakavas +Date: Fri Jan 18 17:06:40 2019 +0200 + + Add validation for empty PutPrivilegeRequest (#37569) + + Return an error to the user if the put privilege api is called with + an empty body (no privileges) + + Resolves: #37561 + +commit ed297b7369e80a79ac5fe43164139ec271163933 +Author: Jason Tedor +Date: Fri Jan 18 08:20:05 2019 -0500 + + Only update response headers if we have a new one (#37590) + + Currently when adding a response header, we do some de-duplication, and + maybe drop the header on the floor if we have reached capacity. Yet, we + still update the thread local tracking the response headers. This is + really expensive because under the hood there is a shared reference that + we synchronize on. In the case of a request processed across many shards + in a tight loop, this contention can be detrimental to performance. We + can avoid updating the thread local in these cases though, when the + response header is duplicate of one that we have already seen, or when + it's dropped on the floor. This commit addresses these performance + issues by avoiding the unnecessary set. + +commit 3a96608b3fea5a855b5ad5f8c6b427f7ade13180 +Author: Christoph Büscher +Date: Fri Jan 18 14:11:18 2019 +0100 + + Remove more include_type_name and types from docs (#37601) + +commit 29d3a708da0623b1ee29c07db99b6db49ba9089b +Author: Tanguy Leroux +Date: Fri Jan 18 10:54:35 2019 +0100 + + Fix BulkWithUpdatesIT and CloseIndexIT + + As of today the Close Index API does its best to close indices, + but closing an index with ongoing recoveries might or might not + be acknowledged depending of the values of the max seq number + and global checkpoint at the time the + TransportVerifyShardBeforeClose action is executed. + + These tests failed because they always expect that the index is + correctly closed on the first try, which is not always the case. + Instead we need to retry the closing until it succeed. + + Closes #37571 + +commit 65e76b3f6f5ae73c91f4f66f9cd2458d13d8c994 +Author: David Turner +Date: Fri Jan 18 09:15:51 2019 +0000 + + Migrate RecoveryFromGatewayIT to Zen2 (#37520) + + * Fixes `testTwoNodeFirstNodeCleared` by manipulating voting config exclusions. + + * Removes `testRecoveryDifferentNodeOrderStartup` since state recovery is now + handled entirely on the elected master, so the order in which the data nodes + start is irrelevant. + +commit 699d88173998ea13dd8ef88d09eaac6fff953d5b +Author: David Turner +Date: Fri Jan 18 09:15:30 2019 +0000 + + Migrate IndicesExistsIT to Zen2 (#37526) + + This test was actually passing, for the wrong reason: it asserts a + `MasterNotDiscoveredException` is thrown, expecting this to be due to a failure + to perform state recovery, but in fact it's thrown because the node is not + correctly bootstrapped. + +commit 25aac4f77fd00f9a181aece5151f7614e0a0563d +Author: Christoph Büscher +Date: Fri Jan 18 09:34:11 2019 +0100 + + Remove `include_type_name` in asciidoc where possible (#37568) + + The "include_type_name" parameter was temporarily introduced in #37285 to facilitate + moving the default parameter setting to "false" in many places in the documentation + code snippets. Most of the places can simply be reverted without causing errors. + In this change I looked for asciidoc files that contained the + "include_type_name=true" addition when creating new indices but didn't look + likey they made use of the "_doc" type for mappings. This is mostly the case + e.g. in the analysis docs where index creating often only contains settings. I + manually corrected the use of types in some places where the docs still used an + explicit type name and not the dummy "_doc" type. + +commit 2f0e0b24265bd94b28235ab76df5e142fa3529ea +Author: Christoph Büscher +Date: Fri Jan 18 09:33:36 2019 +0100 + + Allow indices.get_mapping response parsing without types (#37492) + + This change adds deprecation warning to the indices.get_mapping API in case the + "inlcude_type_name" parameter is set to "true" and changes the parsing code in + GetMappingsResponse to parse the type-less response instead of the one + containing types. As a consequence the HLRC client doesn't need to force + "include_type_name=true" any more and the GetMappingsResponseTests can be + adapted to the new format as well. Also removing some "include_type_name" + parameters in yaml test and docs where not necessary. + +commit 080c07361829e8c852b08745cd3632b8923818cb +Author: Mike Place +Date: Fri Jan 18 00:28:48 2019 -0800 + + Minor docs cleanup (#37595) + + Not all terminals are accessed via SSH. + +commit 62ddc8c7762a363435a2f87f1df47b63b67450ca +Author: Armin Braun +Date: Fri Jan 18 08:36:22 2019 +0100 + + Reenable UnicastZenPingTests#testSimplePings + + * This was muted needlessly, the problem in #26701 only applies to `6.x` + * Relates #26701 + +commit 978c818d0f27b4d0c2e9cbf46a0eccc461a91787 +Author: Tim Brooks +Date: Thu Jan 17 15:31:27 2019 -0700 + + Use RestoreSnapshotRequest in CcrRepositoryIT + + Commit #37535 removed an internal restore request in favor of the + RestoreSnapshotRequest. Commit #37449 added a new test that used the + internal restore request. This commit modifies the new test to use the + RestoreSnapshotRequest. + +commit b6f06a48c0697eba6ccb413806be2658128f2bd1 +Author: Tim Brooks +Date: Thu Jan 17 14:58:46 2019 -0700 + + Implement follower rate limiting for file restore (#37449) + + This is related to #35975. This commit implements rate limiting on the + follower side using a new class `CombinedRateLimiter`. + +commit 381d035cd6cb2e859df956e57b5d033dd19e93b5 +Author: Armin Braun +Date: Thu Jan 17 22:23:23 2019 +0100 + + Remove Redundant RestoreRequest Class (#37535) + + * Same as #37464 but for the restore side + +commit a0c504e4a3ded75352fb25c0a0f18b5c191e102e +Author: Tal Levy +Date: Thu Jan 17 13:21:12 2019 -0800 + + Create specific exception for when snapshots are in progress (#37550) + + delete and close index actions threw IllegalArgumentExceptions + when attempting to run against an index that has a snapshot + in progress. + + This change introduces a dedicated SnapshotInProgressException + for these scenarios. This is done to explicitly signal to clients that + this is the reason the action failed, and it is a retryable error. + + relates to #37541. + +commit 5782a5bbbcf0074a0f8dff8028cc1a1106e36ef0 +Author: James Baiera +Date: Thu Jan 17 15:12:01 2019 -0500 + + Mute UnicastZenPingTests#testSimplePings + + relates #26701 + +commit 6dcb3af4c8300d2d5f5fc9848c2a039956edda5d +Author: Lisa Cawley +Date: Thu Jan 17 10:47:15 2019 -0800 + + [DOCS] Adds size limitation to the get datafeeds APIs (#37578) + +commit 68de2edb14280bafa8cd7f92978237ddb8a6cc90 +Author: Yannick Welsch +Date: Thu Jan 17 19:18:47 2019 +0100 + + Fix assertion at end of forceRefreshes (#37559) + + This commit ensures that we only change refreshListeners to a list if we're actually adding + something to the list. + +commit 6d64a2a90162e90c7626315bfe937c5f91cb996c +Author: Yannick Welsch +Date: Thu Jan 17 17:46:35 2019 +0100 + + Propagate Errors in executors to uncaught exception handler (#36137) + + This is a continuation of #28667 and has as goal to convert all executors to propagate errors to the + uncaught exception handler. Notable missing ones were the direct executor and the scheduler. This + commit also makes it the property of the executor, not the runnable, to ensure this property. A big + part of this commit also consists of vastly improving the test coverage in this area. + +commit a2d9c464b2cf9d0bb640e16321ba61514943473f +Author: Lisa Cawley +Date: Thu Jan 17 08:21:37 2019 -0800 + + [DOCS] Adds limitation to the get jobs API (#37549) + +commit 587034dfa75cf153b22ae95d940a46fdee85dd43 +Author: Jake Landis +Date: Thu Jan 17 09:55:36 2019 -0600 + + Add set_priority action to ILM (#37397) + + This commit adds a set_priority action to the hot, warm, and cold + phases for an ILM policy. This action sets the `index.priority` + on the managed index to allow different priorities between the + hot, warm, and cold recoveries. + + This commit also includes the HLRC and documentation changes. + + closes #36905 + +commit 20ed3dd1a8b9c75e7080f2004e703050223febd6 +Author: Nhat Nguyen +Date: Thu Jan 17 09:59:05 2019 -0500 + + Make recovery source send operations non-blocking (#37503) + + Relates #37458 + +commit 4351a5e5375237b8259fbcbd3ff4d7c59cc2d215 +Author: Jim Ferenczi +Date: Thu Jan 17 15:10:28 2019 +0100 + + Allow field types to optimize phrase prefix queries (#37436) + + This change adds a way to customize how phrase prefix queries should be created + on field types. The match phrase prefix query is exposed in field types in order + to allow optimizations based on the options set on the field. + For instance the text field uses the configured prefix field (if available) to + build a span near that mixes the original field and the prefix field on the last + position. + This change also contains a small refactoring of the match/multi_match query that + simplifies the interactions between the builders. + + Closes #31921 + +commit b85bfd3e1793c71330bb6d2e186d3e6f6edfb74e +Author: Martijn van Groningen +Date: Thu Jan 17 14:04:41 2019 +0100 + + Added fatal_exception field for ccr stats in monitoring mapping. (#37563) + +commit d9fa4e4adaf354d802523e5cf396cf2bfc7f40b4 +Author: Yannick Welsch +Date: Thu Jan 17 13:59:09 2019 +0100 + + Fix testRelocateWhileContinuouslyIndexingAndWaitingForRefresh (#37560) + + This test failed because the refresh at the end of the test is not guaranteed to run before the + indexing is completed, and therefore there's no guarantee that the refresh will free all operations. + This triggers an assertion failure in the test clean-up, which asserts that there are no more pending + operations. + +commit 99b09845da55eaac530517afb1c9187824ced854 +Author: Martijn van Groningen +Date: Thu Jan 17 13:14:06 2019 +0100 + + Moved ccr integration to the package with other ccr integration tests. + +commit 6fe2d6da0391a1da861fc55222c39693388a529b +Author: Yannick Welsch +Date: Thu Jan 17 13:54:48 2019 +0100 + + Mute TransportClientNodesServiceTests#testListenerFailures + + Relates to #37567 + +commit da799306a8c471b02ab4bd8e6e0185933952a7a9 +Author: Martijn van Groningen +Date: Thu Jan 17 11:51:17 2019 +0100 + + Decreased time out in test + + Relates to #37378 + +commit 676e1b1a13588e2d13777242a6ceddecd2f2def4 +Author: Torgeir Thoresen +Date: Thu Jan 17 10:22:49 2019 +0100 + + Fix erroneous docstrings for abstract bulk by scroll request (#37517) + +commit 1686c32ba9825d7998336b942759afaa64fce390 +Author: Marios Trivyzas +Date: Thu Jan 17 10:17:58 2019 +0200 + + SQL: Rename SQL type DATE to DATETIME (#37395) + + * SQL: Rename SQL data type DATE to DATETIME + + SQL data type DATE has only the date part (e.g.: 2019-01-14) + without any time information. Previously the SQL type DATE was + referring to the ES DATE which contains also the time part along + with TZ information. To conform with SQL data types the data type + `DATE` is renamed to `DATETIME`, since it includes also the time, + as a new runtime SQL `DATE` data type will be introduced down the road, + which only contains the date part and meets the SQL standard. + + Closes: #36440 + + * Address comments + +commit b6e5ccaf8a2304892405712cf832d23efdce0940 +Author: Przemyslaw Gomulka +Date: Thu Jan 17 09:10:09 2019 +0100 + + Remove the AbstracLifecycleComponent constructor with Settings (#37523) + + Adding the migration guide and removing the deprecated in 6.x + constructor + + relates #35560 + relates #34488 + +commit 18a3e48a4a7aaff948dc209f7c0798b703d439cc +Author: Jason Tedor +Date: Wed Jan 16 17:19:12 2019 -0500 + + Change file descriptor limit to 65535 (#37537) + + Some systems default to a nofile ulimit of 65535. To reduce the pain of + deploying Elasticsearch to such systems, this commit lowers the required + limit from 65536 to 65535. + +commit 655103de589a36aabb5253942c83faa7a84d219e +Author: Nhat Nguyen +Date: Wed Jan 16 13:17:10 2019 -0500 + + Increase timeout for testAddNewReplicas + + We flush quite often in testAddNewReplicas to create the safe index + commit with gaps in sequence numbers. This test is failing recently + because CI is too slow to complete 5 small flushes in 10 seconds. + + This commit increases timeout for this test and also ensures to always + terminate the background indexing. The latter is to eliminate unrelated + failures if this test fails again. + + Closes #37183 + +commit ecf0de30ba2513fec9c45f8dcc16b22c5c11cd5f +Author: Marios Trivyzas +Date: Wed Jan 16 18:41:10 2019 +0200 + + SQL: Lowercase the datatypes in validation error msgs (#37524) + + To follow the ES convention display the datatypes in lowercase + in error messages thrown during validation if `IN` and conditional + functions. + +commit 4e72f3c5c63f1c8710e56c549a62b27223c1ea4a +Author: Andrey Ershov +Date: Wed Jan 16 11:14:16 2019 -0500 + + DedicatedClusterSnapshotRestoreIT to Zen2 (#37489) + + All tests except testRestorePersistentSettings (renamed to + testExceptionWhenRestoringPersistentSettings) worked fine. + testExceptionWhenRestoringPersistentSettings re-written to use a custom + setting, because "minimum master node" setting is no longer available + in Zen2. It turns out there is no good replacement for "minimum master + node" setting for this test, that's why the custom setting is + introduced. + + Unfortunately, there is #37485 bug and currently + RestoreService does not perform setting validation. That's why the + test is annotated with @AwaitsFix, the idea is to merge this commit and + then fix the issue and enable the test. (The test passes with a simple + fix, that adds a single line to RestoreService). + +commit 2cf4b1863ff97afb4df151690d77981002a99f67 +Author: Marios Trivyzas +Date: Wed Jan 16 18:08:33 2019 +0200 + + SQL: Lowercase es data type (mapping) returned from SQL Commands (#37531) + + To follow the ES convention, convert the es data type, returned as + column `mapping` from SQL Commands, to lowercase. + + Fixes: #37521 + +commit 19d2c29ca6f71ae16620849a19e96d3a26b3d2f4 +Author: Costin Leau +Date: Wed Jan 16 17:28:06 2019 +0200 + + Remove SYS CATALOGS leftover + + Relates #37506 + +commit 1f76b5fc31638e72f3610ca966c1ece19aa3f928 +Author: Costin Leau +Date: Wed Jan 16 17:26:00 2019 +0200 + + SQL: Describe aliases as views (#37496) + + When reporting metadata, several clients have issues with the 'ALIAS' + type. To improve compatibility and be consistent with the ANSI SQL + expectations and because they are similar, aliases targets are now + reported as views. + + Close #37422 + +commit 3d8c04659c8ea849d2621cd1b8edb29882d1eed2 +Author: Jack Conradson +Date: Wed Jan 16 07:05:09 2019 -0800 + + Deprecate _type from LeafDocLookup (#37491) + + * Deprecate _type from LeafDocLookup + + * Response to PR comments. + + * Response to PR comments. + +commit 0b5af276a8b3d4cf3e94ff6ba4264d1bd1f0334c +Author: Tim Brooks +Date: Wed Jan 16 07:52:38 2019 -0700 + + Allow system privilege to execute proxied actions (#37508) + + Currently all proxied actions are denied for the `SystemPrivilege`. + Unfortunately, there are use cases (CCR) where we would like to proxy + actions to a remote node that are normally performed by the + system context. This commit allows the system context to perform + proxy actions if they are actions that the system context is normally + allowed to execute. + +commit 86697f23a309f1648ccc257bf438f51a95d8622d +Author: Michael Basnight +Date: Wed Jan 16 08:50:03 2019 -0600 + + Update Put Watch to allow unknown fields (#37494) + + PutWatchResponse did not allow unknown fields. This commit fixes the + test and ConstructingObjectParser such that it does now allow unknown + fields. + + Relates #36938 + +commit 0160ba2539d3b2335e8bc825972128403a449f9c +Author: Nhat Nguyen +Date: Wed Jan 16 09:18:52 2019 -0500 + + AwaitsFix testAddNewReplicas + + Tracked at #37183 + +commit 659326fdd63097626a82d2f004a446482a042cd7 +Author: Andrei Stefan +Date: Wed Jan 16 16:28:46 2019 +0200 + + SQL: Add protocol tests and remove jdbc_type from drivers response (#37516) + +commit 8932750efbafb54a16e313bc613209bdc57702d2 +Author: Marios Trivyzas +Date: Wed Jan 16 16:16:21 2019 +0200 + + SQL: [Docs] Add an ES-SQL column for data types (#37529) + + In order to distinguish the ES-SQL type from the standard SQL type + add a new ES-SQL column that will make clear this distingstion, + e.g.: datetime vs TIMSTAMP + + Fixes: #37519 + +commit 9d8afe68a534741a4cc9357b6ab4c719e46011eb +Author: Adrien Grand +Date: Wed Jan 16 14:01:09 2019 +0100 + + IndexMetaData#mappingOrDefault doesn't need to take a type argument. (#37480) + + Currently it takes a type, but this isn't really needed now that indices can + have at most one type. The only downside is that we might return a different + error when trying to index into a type that doesnt't exist yet. + +commit 21a88d5505835592dcaea1e56690c94c723daa4e +Author: Armin Braun +Date: Wed Jan 16 13:57:16 2019 +0100 + + Simplify + Cleanup Dead Code in Settings (#37341) + + * Remove dead code + * Simplify some overly complex code, this class is long enough already + +commit 687978b7d167d4c20a29861b2558b2749c349529 +Author: Jason Tedor +Date: Wed Jan 16 07:29:25 2019 -0500 + + Reject all requests that have an unconsumed body (#37504) + + This commit removes some leniency from REST handling where we move to + reject all requests that have a body where the body is not used during + the course of handling the request. For example, + + DELETE /index + { + "query" : { + "term" : { + "field" : "value" + } + } + } + + is now rejected. + +commit 75410dc632469ef602760801b35e1eae5e2375d4 +Author: David Kyle +Date: Wed Jan 16 08:38:46 2019 +0000 + + [Ml] Prevent config snapshot failure blocking migration (#37493) + +commit 347cbaf0ed10c421a7887d424a50064b601fe2ec +Author: Dimitrios Liappis +Date: Wed Jan 16 13:06:29 2019 +0200 + + Fix line length for aliases and remove suppression (#37455) + + Relates #34884 + +commit 6d99e790b3ee78f99f4810ff79b38488e43a146d +Author: Tim Vernum +Date: Wed Jan 16 21:52:17 2019 +1100 + + Add SSL Configuration Library (#37287) + + This introduces a new ssl-config library that can parse + and validate SSL/TLS settings and files. + + It supports the standard configuration settings as used in the + Elastic Stack such as "ssl.verification_mode" and + "ssl.certificate_authorities" as well as all file formats used + in other parts of Elasticsearch security (such as PEM, JKS, + PKCS#12, PKCS#8, et al). + +commit 023bb2f1e473f4ad009e22c1947faf844d39dbad +Author: Costin Leau +Date: Wed Jan 16 12:36:35 2019 +0200 + + SQL: Remove slightly used meta commands (#37506) + + Remove SYS CATALOGS and SYS TABLE TYPES as they are a subset of SYS + TABLES (and thus somewhat redundant) and used only by JDBC. + + Close #37409 + +commit 5a5e44d1de98f79ae19a6b86af0b0fdf12f0ef72 +Author: Armin Braun +Date: Wed Jan 16 11:08:48 2019 +0100 + + Simplify Snapshot Create Request Handling (#37464) + + * The internal create request is absolutely redundant, the only difference to the transport request is that we resolved the snapshot + name when moving from the transport to the internal version + * Removed it and passed the transport request into the snapshot service instead + * nicer way of resolve snapshot name in callback + +commit 5e94f384c41fecc94c412c3162863c93895dfe4c +Author: Przemyslaw Gomulka +Date: Wed Jan 16 09:05:30 2019 +0100 + + Remove the use of AbstracLifecycleComponent constructor #37488 (#37488) + + The AbstracLifecycleComponent used to extend AbstractComponent, so it had to pass settings to the constractor of its supper class. + It no longer extends the AbstractComponent so there is no need for this constructor + There is also no need for AbstracLifecycleComponent subclasses to have Settings in their constructors if they were only passing it over to super constructor. + This is part 1. which will be backported to 6.x with a migration guide/deprecation log. + part 2 will have this constructor removed in 7 + relates #35560 + + relates #34488 + +commit 15d1b904a1a053c3d26072c604fa2ff0026bc87e +Author: Hendrik Muhs +Date: Wed Jan 16 08:10:13 2019 +0100 + + [ML] log minimum diskspace setting if forecast fails due to insufficient d… (#37486) + + log minimum disk space setting if forecast fails due to insufficient disk space + +commit 987576b0133b20ea338a4f7022ac8eaad9805420 +Author: Jay Modi +Date: Tue Jan 15 14:03:21 2019 -0700 + + Consistently use loopback address for ssl profile (#37487) + + This change fixes failures in the SslMultiPortTests where we attempt to + connect to a profile on a port it is listening on but the connection + fails. The failure is due to the profile being bound to multiple + addresses and randomization will pick one of these addresses to + determine the listening port. However, the address we get the port for + may not be the address we are actually connecting to. In order to + resolve this, the test now sets the bind host for profiles to the + loopback address and uses the same address for connecting. + + Closes #37481 + +commit 0a3bff2ca98cf407fcb889823883b9df1c980778 +Author: Julie Tibshirani +Date: Tue Jan 15 12:38:32 2019 -0800 + + Only log one types warning per bulk search request. (#37446) + +commit e7511753a95124b1c10f4cf13253e5871c971686 +Author: Alpar Torok +Date: Tue Jan 15 21:20:19 2019 +0200 + + Upgrade to Gradle 5.1.1 (#37410) + + Fixes mem leak in the daemon. + +commit 17b0aa3f316319eeb8af1ebf1155f4d514af2eaa +Author: Lisa Cawley +Date: Tue Jan 15 11:11:01 2019 -0800 + + [DOCS] Intro for adding nodes (#37202) + +commit 03cacdf16e10894bab7861406ce708537783ab8c +Author: Lisa Cawley +Date: Tue Jan 15 10:53:29 2019 -0800 + + [DOCS] Adds CCR screenshots (#37439) + +commit 42fd68ed389b6e3c72d5dd2108a5dc7fbe429399 +Author: Andrey Ershov +Date: Tue Jan 15 13:33:25 2019 -0500 + + Use GatewayMetaState in CoordinatorTests rarely (#36897) + + This commit adds one more underlying implementation of MockPersistedState. + Previously only InMemoryPersistentState was used, not GatewayMetaState + is used rarely. + When adding GatewayMetaState support the main question was: do we want to + emulate exceptions as we do today in MockPersistedState before + delegating to GatewayMetaState or do we want these exceptions to + propagate from the lower level, i.e. file system exceptions? + On the one hand, lower level exception propagation is already tested in + GatewayMetaStateTests, so this won't improve the coverage. + On the other hand, the benefit of low-level exceptions is to see how all these + components work in conjunction. Finally, we abandoned the idea of low-level + exceptions because we don't have a way to deal with IOError today in + CoordinatorTests, but hacking GatewayMetaState not to throw + IOError seems unnatural. + So MockPersistedState rarely throws an exception before delegating to + GatewayMetaState, which is not supposed to throw the exception. + + This commit required two changes: + + Move GatewayMetaStateUT to upper-level from + GatewayMetaStatePersistedStateTests, because otherwise, it's not easy + to construct GatewayMetaState instance in CoordinatorTests. + Move addition of STATE_NOT_RECOVERED_BLOCK from GatewayMetaState + constructor to GatewayMetaState.applyClusterUpdaters, because + CoordinatorTests class assumes that there is no such block and most of + them fail. + +commit f8d80dff7c03774da3c4e2a5d4e7cfd15cf0747e +Author: Jim Ferenczi +Date: Tue Jan 15 19:27:31 2019 +0100 + + Fix duplicate removal when merging completion suggestions (#36996) + + The completion suggester ignores the original weight of the suggestion when duplicates are removed. This change fixes this bug and keeps the best weighted suggestion among the duplicates. It also removes the custom implementation of the top docs suggest collector now that https://issues.apache.org/jira/browse/LUCENE-8529 is committed in Lucene. + + Closes #35836 + +commit bea46f7b52f9ce3e731773f87558c456fe30a5d5 +Author: David Kyle +Date: Tue Jan 15 18:21:39 2019 +0000 + + [ML] Migrate unallocated jobs and datafeeds (#37430) + + Migrate ml job and datafeed config of open jobs and update + the parameters of the persistent tasks as they become unallocated + during a rolling upgrade. Block allocation of ml persistent tasks + until the configs are migrated. + +commit 2cf7a8016feed9096ca2994872244dbfac0706cb +Author: Ryan Ernst +Date: Tue Jan 15 10:19:32 2019 -0800 + + Packaging: Remove permission editing in postinst (#37242) + + This commit removes permission editing commands from the postinst + scriptlet. Instead, we now fully configure the owner/group (as well as + sticky bit) for these files and directories. + + closes #37143 + +commit 6647122f1c22c484ded40f45c65d24366a84abda +Author: Nhat Nguyen +Date: Tue Jan 15 13:17:25 2019 -0500 + + Prepare to make send translog of recovery non-blocking (#37458) + + This commit prepares the required infra to make send a translog snapshot + of the recovery source non-blocking. I'll make a follow-up to make the send + snapshot method non-blocking. + + Relates #37291 + +commit 02d4d8b4099b071b0f846a26b7c02adeb19a2e90 +Author: Andrey Ershov +Date: Tue Jan 15 13:09:48 2019 -0500 + + MinimumMasterNodesIT changed for Zen2 (#37428) + + There were 5 tests in MinimumMasterNodesIT. 2 of them removed, 3 of + them changed and renamed. + 1) testSimpleMinimumMasterNodes -> testTwoNodesNoMasterBlock. The + flow of this test is left intact but in order to make it work on + Zen2, additional work for the cluster bootstrapping and voting + exclusions is needed. + 2) testDynamicUpdateMinimumMasterNodes -> removed, there is nothing + that corresponds to the dynamic change of the minimum master nodes + setting. + 3) testCanNotBringClusterDown -> removed, it also plays with changing + minimum master nodes dynamically. + 4) testMultipleNodesShutdownNonMasterNodes -> + testThreeNodesNoMasterBlock. Previously this test was checking that + there would be no master block, if min_master_nodes=3 and 4 nodes are + started, then 2 nodes are brought down. Zen2 dynamically accommodates + to the number of nodes in the cluster, so it's possible that there + still will be a master in 2 nodes cluster. For Zen2, we start up 3 + nodes. And shut down 2 of them (w/o voting exclusions), which results + in no master block. + 5) testCanNotPublishWithoutMinMastNodes -> + testCanNotCommitStateThreeNodes. Test flow is not changed. But + previously there was no check that nodes in the bigger part of + network partition will elect the master, before healing the network + partition. For Zen2 it does not work, because persistent setting + addition is accepted on the old master and if it's elected new master + again, this setting will appear in the cluster state. + + Also, I have a feeling that we need to remove this class, but could not + come up with a good name. + +commit 63793499bdce1d2f6290d843efd9bfc4bd3809d4 +Author: Dimitrios Liappis +Date: Tue Jan 15 19:57:24 2019 +0200 + + Fix line length for `node` and remove suppresion (#37454) + + Relates #34884 + +commit a2a40c50a0db185e34c317a7cf666de7717be707 +Author: David Turner +Date: Tue Jan 15 17:32:08 2019 +0000 + + Report terms and version if cluster does not form (#37473) + + Adds the node's current term and the term and version of the the last-accepted + cluster state to the message reported by the `ClusterFormationFailureHelper`, + since these values may be of importance when tracking down a cluster formation + failure. + +commit 7c11b05c2877114c8003aa233587d8d3021ea013 +Author: David Kyle +Date: Tue Jan 15 17:19:58 2019 +0000 + + [ML] Remove unused code from the JIndex project (#37477) + +commit 44e83f30e2e11c22f9bd49ef9354e9b97f261f78 +Author: Lisa Cawley +Date: Tue Jan 15 08:46:36 2019 -0800 + + [DOCS] Edits rollup API description (#37444) + +commit 6129e9d9ddeb38be1ba88b2451503c7faa35db32 +Author: Marios Trivyzas +Date: Tue Jan 15 18:24:42 2019 +0200 + + Revert "[TEST] Muted TokenBackwardsCompatibilityIT.*" + + This reverts commit 65e42ab63bb6e764ea9f547b42bf82b3ef465881. + + The test is only failing in 6.x not master. + +commit 68e2d36fa301de9d85ec01861cbc8f706296d07c +Author: Nhat Nguyen +Date: Tue Jan 15 11:18:55 2019 -0500 + + Adjust bwc version for max_concurrent_file_chunks + + Relates #36981 + +commit 65e42ab63bb6e764ea9f547b42bf82b3ef465881 +Author: Marios Trivyzas +Date: Tue Jan 15 18:04:31 2019 +0200 + + [TEST] Muted TokenBackwardsCompatibilityIT.* + + Relates to #37379 + +commit 6f91f06d8679bbf2bfc7563d38cc500d83deba1c +Author: Igor Motov +Date: Tue Jan 15 10:52:46 2019 -0500 + + Geo: Adds a set of no dependency geo classes for JDBC driver (#36477) + + Adds a set of geo classes to represent geo data in the JDBC driver and + to be used as an intermediate format to pass geo shapes for indexing + and query generation in #35320. + + Relates to #35767 and #35320 + +commit 0b396a0c5e5f5a463ccc629a7826a78389b4557f +Author: Luca Cavanna +Date: Tue Jan 15 16:42:55 2019 +0100 + + Restore assertion on discount overlaps in SimilarityTests + + This assertion was commented out as the getDiscountOverlaps getter was + missing from LegacyBm25Similarity. That has been fixed in lucene. + +commit 1a1dbf705faaff44d40bd24168ecce15fbfe9540 +Author: Julie Tibshirani +Date: Tue Jan 15 07:32:47 2019 -0800 + + Make sure to use the resolved type in DocumentMapperService#extractMappings. (#37451) + + * Pull out a shared method MapperService#resolveDocumentType. + * Make sure to resolve the type when extracting the mappings. + + Addresses #36811. + +commit 9554b2fecb27b205d5e960e3fb12e0901b2badf9 +Author: Martijn van Groningen +Date: Tue Jan 15 16:24:19 2019 +0100 + + When removing an AutoFollower also mark it as removed. (#37402) + + Currently when there are no more auto follow patterns for a remote cluster then + the AutoFollower instance for this remote cluster will be removed. If + a new auto follow pattern for this remote cluster gets added quickly enough + after the last delete then there may be two AutoFollower instance running + for this remote cluster instead of one. + + Each AutoFollower instance stops automatically after it sees in the + start() method that there are no more auto follow patterns for the + remote cluster it is tracking. However when an auto follow pattern + gets removed and then added back quickly enough then old AutoFollower + may never detect that at some point there were no auto follow patterns + for the remote cluster it is monitoring. The creation and removal of + an AutoFollower instance happens independently in the `updateAutoFollowers()` + as part of a cluster state update. + + By adding the `removed` field, an AutoFollower instance will not miss the + fact there were no auto follow patterns at some point in time. The + `updateAutoFollowers()` method now marks an AutoFollower instance as + removed when it sees that there are no more patterns for a remote cluster. + The updateAutoFollowers() method can then safely start a new AutoFollower + instance. + + Relates to #36761 + +commit 3cc8f39532dac9768704d07c7611e6b1a162dbdc +Author: Fabricio Archanjo Fonseca +Date: Tue Jan 15 13:06:32 2019 -0200 + + New mapping signature and mapping string source fixed. (#37401) + + * New mapping signature and mapping string source fixed. + + * Keep compatibility with CreateIndexRequest class. + +commit e71542ffd3e31335c8fc315d6f31e82d29ead4df +Author: niloct +Date: Tue Jan 15 13:04:32 2019 -0200 + + Update delete-by-query.asciidoc (#37370) + + Tried my best to clarify sentence on `_delete_by_query` docs. + +commit a56aa4f076959f834c226c911996053f95a955a6 +Author: Jay Modi +Date: Tue Jan 15 07:52:58 2019 -0700 + + Remove SslNullCipherTests from codebase (#37431) + + This change deletes the SslNullCipherTests from our codebase since it + will have issues with newer JDK versions and it is essentially testing + JDK functionality rather than our own. The upstream JDK issue for + disabling these ciphers by default is + https://bugs.openjdk.java.net/browse/JDK-8212823. + + Closes #37403 + +commit 7cdf7f882b0debd42964a0d4300406cd5fc31297 +Author: David Roberts +Date: Tue Jan 15 14:19:05 2019 +0000 + + [ML] Fix ML datafeed CCS with wildcarded cluster name (#37470) + + The test that remote clusters used by ML datafeeds have + a license that allows ML was not accounting for the + possibility that the remote cluster name could be + wildcarded. This change fixes that omission. + + Fixes #36228 + +commit e848388865e8936b5befd4586a910dbb50d3a307 +Author: Tanguy Leroux +Date: Tue Jan 15 15:03:09 2019 +0100 + + Fix SourceOnlySnapshotIT (#37461) + + The SourceOnlySnapshotIT class tests a source only repository + using the following scenario: + starts a master node + starts a data node + creates a source only repository + creates an index with documents + snapshots the index to the source only repository + deletes the index + stops the data node + starts a new data node + restores the index + + Thanks to ESIntegTestCase the index is sometimes created using a custom + data path. With such a setting, when a shard is assigned to one of the data + node of the cluster the shard path is resolved using the index custom data + path and the node's lock id by the NodeEnvironment#resolveCustomLocation(). + + It should work nicely but in SourceOnlySnapshotIT.snashotAndRestore(), b + efore the change in this PR, the last data node was restarted using a different + path.home. At startup time this node was assigned a node lock based on other + locks in the data directory of this temporary path.home which is empty. So it + always got the 0 lock id. And when this new data node is assigned a shard for + the index and resolves it against the index custom data path, it also uses the + node lock id 0 which conflicts with another node of the cluster, resulting in + various errors with the most obvious one being LockObtainFailedException. + + This commit removes the temporary home path for the last data node so that it + uses the same path home as other nodes of the cluster and then got assigned + a correct node lock id at startup. + + Closes #36330 + Closes #36276 + +commit a88c050a05cc2727225497cbe2ce16f0c5a1766f +Author: Albert Zaharovits +Date: Tue Jan 15 14:29:32 2019 +0200 + + Docs be explicit on how to turn off deprecated auditing (#37316) + + Just be explicit about turning off the deprecated audit log appender + because we really want people to turn it off. + +commit 19fc59f0892bce149a7f609483bb97e0e57fa98e +Author: Dimitrios Liappis +Date: Tue Jan 15 14:18:15 2019 +0200 + + Fix line length for monitor and remove suppressions (#37456) + + Relates #34884 + +commit 23ae9808bac69f1f4356af152c55de2632fcfed2 +Author: Tanguy Leroux +Date: Tue Jan 15 12:48:21 2019 +0100 + + Fix IndexShardTestCase.recoverReplica(IndexShard, IndexShard, boolean) (#37414) + + This commit fixes the IndexShardTestCase.recoverReplica(IndexShard, IndexShard, boolean) + method where the startReplica parameter was not correctly propagated and the value + true always used instead. + +commit f7eb517eb8fcd2109e776ffb354ed24240683967 +Author: Boaz Leskes +Date: Tue Jan 15 11:43:09 2019 +0100 + + Update the Flush API documentation (#33551) + + The semantics of the API changed considerably since the documentation was written. + + The main change is to remove references to memory reduction (this is related to refresh). + Instead, flush refers to recovery times. I also removed the references to trimming the translog + as the translog may be required for other purposes (operation history for ops based recovery + and complement ongoing file based recoveries). + + Closes #32869 + +commit d6a104f52bc047274b93a9274933b77fef004bba +Author: Marios Trivyzas +Date: Tue Jan 15 11:51:45 2019 +0200 + + [TEST] Muted testDifferentRolesMaintainPathOnRestart + + Relates to #37462 + +commit 147c5e65d35f4397b9fbeef56a8104a8c80ec741 +Author: Simon Willnauer +Date: Tue Jan 15 09:39:53 2019 +0100 + + Remove dead code from ShardSearchStats (#37421) + + The clear methodsa are unused and unsafe at this point. This commit + removes the dead code. + +commit bf49f54456e3b9ad2bb5bd5b739b0306a4f437b2 +Author: Nhat Nguyen +Date: Tue Jan 15 03:07:56 2019 -0500 + + Simplify testSendSnapshotSendsOps (#37445) + + The test testSendSnapshotSendsOps is currently using a mock instance of + RecoveryTargetHandler which will be hard to modify when we make the + RecoveryTargetHandler non-blocking. This commit prepares for the + incoming changes by replacing the mock instance with a stub. + +commit b594e81c86b1032cb79e59a277fe9de49458ca62 +Author: Marios Trivyzas +Date: Tue Jan 15 09:41:41 2019 +0200 + + SQL: Fix issue with field names containing "." (#37364) + + Adjust FieldExtractor to handle fields which contain `.` in their + name, regardless where they fall in, in the document hierarchy. E.g.: + + ``` + { + "a.b": "Elastic Search" + } + + { + "a": { + "b.c": "Elastic Search" + } + } + + { + "a.b": { + "c": { + "d.e" : "Elastic Search" + } + } + } + ``` + + Fixes: #37128 + +commit b97245cfcd927d2326710de34b66b6cfd4b018a8 +Author: Tim Vernum +Date: Tue Jan 15 18:20:22 2019 +1100 + + Restore lost @Inject annotation (#37452) + + The Inject Annotation was removed from IndicesClusterStateService as + part of reformatting in e11a32e, but this causes CreationException on + cluster startup. + +commit 43bfdd32eea161a9084d7b49b12261ea32a7983c +Author: Jason Tedor +Date: Mon Jan 14 21:54:42 2019 -0500 + + Add run under primary permit method (#37440) + + This commit adds a simple method for executing a runnable against a + shard under a primary permit. Today there is only a single caller for + this method, but this there are two upcoming use-cases for which having + this method will help keep the code simpler. + +commit e11a32eda8f6182d3ebf98955e3ec39daf6a398c +Author: Jason Tedor +Date: Mon Jan 14 21:43:56 2019 -0500 + + Reformat some classes in the index universe + + This commit reformats some classes in the index universe with the + purpose of breaking some long method definitions and invocations into a + line per parameter. This has the advantage that for an upcoming change + to these definitions and invocations, the diff for that change will be a + single line per definition or invocation. That makes these sorts of + changes easier to read. + +commit c0368a20864dbc8780508b88c668d8730c21c586 +Author: Michael Basnight +Date: Mon Jan 14 20:44:21 2019 -0600 + + [DOCS] Add watcher context examples (#36565) + +commit 3bc0711b908b3f68d3d0b52fae43f42aba8b8a92 +Author: Jason Tedor +Date: Mon Jan 14 21:28:28 2019 -0500 + + Add simple method to write collection of writeables (#37448) + + This commit adds a simple convenience method for writing a collection of + writeables, and replaces existing call sites with the new method. + +commit eb86b9f284ae0f272b51218efe68f36e294dedbc +Author: Jason Tedor +Date: Mon Jan 14 21:15:35 2019 -0500 + + Fix retention lease commit test + + This commit fixes an issue with testing committed retention leases when + they are not any retention leases (a deliberate edge case). + + Closes #37420 + +commit 74640d0ba7cbb0f6ffd78ff116990ae8d8e5cfde +Author: Jason Tedor +Date: Mon Jan 14 21:06:44 2019 -0500 + + Introduce retention lease serialization (#37447) + + This commit is a simple introduction of the serialization of retention + leases, which will be needed when they are sent across the wire while + synchronizing retention leases to replicas. + +commit cd78565acfe307549974b272138d2a3c9255e7ff +Author: Michael Basnight +Date: Mon Jan 14 19:50:20 2019 -0600 + + Update Delete Watch to allow unknown fields (#37435) + + DeleteWatchResponse did not allow unknown fields. This commit fixes the + test and ConstructingObjectParser such that it does now allow unknown + fields. + + Relates #36938 + +commit 397f315f5691c6f57a4cf3a2199315eb693a819d +Author: Nhat Nguyen +Date: Mon Jan 14 18:20:54 2019 -0500 + + Make finalize step of recovery source non-blocking (#37388) + + Relates #37291 + +commit 36a3b84fc968f9aec5c3482dee69a1e53a7ac94a +Author: Julie Tibshirani +Date: Mon Jan 14 13:08:01 2019 -0800 + + Update the default for include_type_name to false. (#37285) + + * Default include_type_name to false for get and put mappings. + + * Default include_type_name to false for get field mappings. + + * Add a constant for the default include_type_name value. + + * Default include_type_name to false for get and put index templates. + + * Default include_type_name to false for create index. + + * Update create index calls in REST documentation to use include_type_name=true. + + * Some minor clean-ups around the get index API. + + * In REST tests, use include_type_name=true by default for index creation. + + * Make sure to use 'expression == false'. + + * Clarify the different IndexTemplateMetaData toXContent methods. + + * Fix FullClusterRestartIT#testSnapshotRestore. + + * Fix the ml_anomalies_default_mappings test. + + * Fix GetFieldMappingsResponseTests and GetIndexTemplateResponseTests. + + We make sure to specify include_type_name=true during xContent parsing, + so we continue to test the legacy typed responses. XContent generation + for the typeless responses is currently only covered by REST tests, + but we will be adding unit test coverage for these as we implement + each typeless API in the Java HLRC. + + This commit also refactors GetMappingsResponse to follow the same appraoch + as the other mappings-related responses, where we read include_type_name + out of the xContent params, instead of creating a second toXContent method. + This gives better consistency in the response parsing code. + + * Fix more REST tests. + + * Improve some wording in the create index documentation. + + * Add a note about types removal in the create index docs. + + * Fix SmokeTestMonitoringWithSecurityIT#testHTTPExporterWithSSL. + + * Make sure to mention include_type_name in the REST docs for affected APIs. + + * Make sure to use 'expression == false' in FullClusterRestartIT. + + * Mention include_type_name in the REST templates docs. + +commit f3edbe291146d92888087c6458a08d2e4cbb602d +Author: Jay Modi +Date: Mon Jan 14 14:06:22 2019 -0700 + + Security: remove SSL settings fallback (#36846) + + This commit removes the fallback for SSL settings. While this may be + seen as a non user friendly change, the intention behind this change + is to simplify the reasoning needed to understand what is actually + being used for a given SSL configuration. Each configuration now needs + to be explicitly specified as there is no global configuration or + fallback to some other configuration. + + Closes #29797 + +commit b86621c157eb46083f985ae830b56dfadbea6ea5 +Author: Shaunak Kashyap +Date: Mon Jan 14 12:41:10 2019 -0800 + + Adding mapping for hostname field (#37288) + + This new `hostname` field is meant to be a replacement for its sibling `name` field. See https://github.com/elastic/beats/pull/9943, particularly https://github.com/elastic/beats/pull/9943#discussion_r245932581. + + This PR simply adds the new field (`hostname`) to the mapping without removing the old one (`name`), because a user might be running an older-version Beat (without this field rename in it) with a newer-version Monitoring ES cluster (with this PR's change in it). + + AFAICT the Monitoring UI isn't currently using the `name` field so no changes are necessary there yet. If it decides to start using the `name` field, it will also want to look at the value of the `hostname` field. + +commit 1e3702da0b54e32b0e4aaa8ea1955cbbe4215afa +Author: Nhat Nguyen +Date: Mon Jan 14 11:40:11 2019 -0500 + + Relax assertSameDocIdsOnShards assertion + + If the checking node no longer holds the shard copy, the assertion + assertSameDocIdsOnShards might fail. This is too harsh since the + assertion is to ensure the consistency between active copies. + +commit 15aa3764a47bf6c9fd99dc5e0713acb12105d55c +Author: Nhat Nguyen +Date: Mon Jan 14 15:14:46 2019 -0500 + + Reduce recovery time with compress or secure transport (#36981) + + Today file-chunks are sent sequentially one by one in peer-recovery. This is a + correct choice since the implementation is straightforward and recovery is + network bound in most of the time. However, if the connection is encrypted, we + might not be able to saturate the network pipe because encrypting/decrypting + are cpu bound rather than network-bound. + + With this commit, a source node can send multiple (default to 2) file-chunks + without waiting for the acknowledgments from the target. + + Below are the benchmark results for PMC and NYC_taxis. + + - PMC (20.2 GB) + + | Transport | Baseline | chunks=1 | chunks=2 | chunks=3 | chunks=4 | + | ----------| ---------| -------- | -------- | -------- | -------- | + | Plain | 184s | 137s | 106s | 105s | 106s | + | TLS | 346s | 294s | 176s | 153s | 117s | + | Compress | 1556s | 1407s | 1193s | 1183s | 1211s | + + - NYC_Taxis (38.6GB) + + | Transport | Baseline | chunks=1 | chunks=2 | chunks=3 | chunks=4 | + | ----------| ---------| ---------| ---------| ---------| -------- | + | Plain | 321s | 249s | 191s | * | * | + | TLS | 618s | 539s | 323s | 290s | 213s | + | Compress | 2622s | 2421s | 2018s | 2029s | n/a | + + Relates #33844 + +commit 5c68338a1c4bd3a41d566c845c2c3ee980c07d55 +Author: Tim Brooks +Date: Mon Jan 14 13:07:55 2019 -0700 + + Implement ccr file restore (#37130) + + This is related to #35975. It implements a file based restore in the + CcrRepository. The restore transfers files from the leader cluster + to the follower cluster. It does not implement any advanced resiliency + features at the moment. Any request failure will end the restore. + +commit c801b89072a8b9d35c3a40d247ede51834f99926 +Author: Christoph Büscher +Date: Mon Jan 14 20:39:04 2019 +0100 + + Fix Eclipse specific compilation issue (#37419) + + Without pulling out the supplier function to the enclosing class, Eclipse 4.8 + complains with the following error "No enclosing instance of type + CoordinatorTests.Cluster is available due to some intermediate constructor + invocation" + +commit 92c6c98e8d8d4131f1c90b13056bc07c83d184b8 +Author: markharwood +Date: Mon Jan 14 17:51:49 2019 +0000 + + Performance fix. Reduce deprecation calls for the same bulk request (#37415) + + DeprecationLogger has warning de-duplication logic but it is expensive to run as it involves parsing existing warning headers. This PR changes the upstream bulk indexing code to do its own "event thinning" rather than relying on DeprecationLogger's trimming. + Closes #37411 + +commit 2ee55a50bf83749b02b0913fa989f0d8aafaa74f +Author: David Kyle +Date: Mon Jan 14 16:39:47 2019 +0000 + + [ML] Use String rep of Version in map for serialisation (#37416) + +commit 033e67fa593ab831ee008824c03f6c2f161842b6 +Author: Armin Braun +Date: Mon Jan 14 16:22:44 2019 +0100 + + Cleanup Deadcode in Rest Tests (#37418) + + * Either dead code outright or redundant overrides removed + +commit 1abe5df09c7635175e0b6216bb58563d6c879eac +Author: David Kyle +Date: Mon Jan 14 14:17:11 2019 +0000 + + Mute IndexShardRetentionLeaseTests.testCommit #37420 + +commit de852765d6bd7d06497cb15136c8b61bfa93744c +Author: Martijn van Groningen +Date: Mon Jan 14 14:27:26 2019 +0100 + + unmuted test + + Relates to #37014 + +commit abe35fb99bc6d3adf7c68278044437216d647315 +Author: Daniel Mitterdorfer +Date: Mon Jan 14 13:44:32 2019 +0100 + + Remove unused index store in directory service + + With this commit we remove the unused field `indexStore` from all + implementations of `FsDirectoryService`. + + Relates #37097 + +commit 07dc8c7eee8f329c81e514c61c5b6edaaa1ad890 +Author: Tanguy Leroux +Date: Mon Jan 14 13:14:36 2019 +0100 + + Improve CloseWhileRelocatingShardsIT (#37348) + +commit 6ca076bf7467f13d7807b089f549ac429b45e8fc +Author: Tanguy Leroux +Date: Mon Jan 14 13:13:15 2019 +0100 + + Fix ClusterBlock serialization and Close Index API logic after backport to 6.x (#37360) + + This commit changes the versions in the serialization logic of ClusterBlock + after the backport to 6.x of the Close Index API refactoring (#37359). + +commit 87f91485805d0f3eb54dd6eacaac5994f6faa090 +Author: Georgi Ivanov +Date: Mon Jan 14 13:03:00 2019 +0100 + + Update the scroll example in the docs (#37394) + + Update the scroll example ascii and Java docs, so it is more clear when to + consume the scroll documents. Before this change the user could loose + the first results if one uses copy & paste. + +commit 4b1368183d741bb1f5818d24b638eb38e0954d24 +Author: Jiyu-Zhang-Zendesk <36034626+Jiyu-Zhang-Zendesk@users.noreply.github.com> +Date: Mon Jan 14 11:20:25 2019 +0100 + + Update analysis.asciidoc (#37404) + + STConvert plugin is made by Medcl to convert between Simplified Chinese and Traditional Chinese. It's widely used by the Search Community for Chinese + +commit 89b45f1fc6ed7bb46df5071f7ad34d80e6e87761 +Author: Christoph Büscher +Date: Mon Jan 14 11:14:38 2019 +0100 + + Remove deprecated pipeline request contructors (#37366) + + The constructors in PutPipelineRequest and SimulatePipelineRequest that guess + the xContent type from the provided source are deprecated since 6.0 and each + have a counterpart that takes the xContent type as an explicit argument. + Removing these ctors together with the builders and methods in + ClusterAdminClient that don't have the xContent type as argument. + +commit d44a6f9fbcc4f71116a5b6eb0573a786d67a58ae +Author: Nhat Nguyen +Date: Mon Jan 14 03:54:34 2019 -0500 + + Simplify SyncedFlushService flow with StepListener (#37383) + + Today the SyncedFlushService flow is written with multiple nested + callbacks which are hard to read. This commit replaces them with + sequential step listeners. + +commit 374e24c7fd09dd80437138e6ca25803abd8469f3 +Author: Ioannis Kakavas +Date: Mon Jan 14 10:50:24 2019 +0200 + + Mute SslNullCipherTests on JDK12 + + JDK12 doesn't support NULL cipher for TLS by default. This commit + mutes these tests on JDK12 until we decide whether we need to keep + or remove them + +commit d54f88f62c1ab9f0d25f90c49b8184a1e0eb4a65 +Author: Luca Cavanna +Date: Mon Jan 14 08:32:45 2019 +0100 + + Remove unused empty constructors from suggestions classes (#37295) + + We recently migrated suggestions to `Writeable`. That allows us to also + clean up empty constructors and methods that called them as they are no + longer needed. They are replaced by constructors that accept a + `StreamInput` instance. + +commit a566bacbc8e263a95e0975ff2639e095d4a74af8 +Author: Alpar Torok +Date: Sun Jan 13 19:33:39 2019 +0200 + + Upgrade ASM for java 12 compatability (#37385) + + Closes #37371 + +commit 6fd57d90dad0df06aece223e5e8e672b8d0bc908 +Author: Albert Zaharovits +Date: Sun Jan 13 15:26:23 2019 +0200 + + Security Audit includes HTTP method for requests (#37322) + + Adds another field, named "request.method", to the structured logfile audit. + This field is present for all events associated with a REST request (not a + transport request) and the value is one of GET, POST, PUT, DELETE, OPTIONS, + HEAD, PATCH, TRACE and CONNECT. + +commit 96cfa000a50a6bcee33298e2d2afd9e560684a8a +Author: Peter Dyson +Date: Sun Jan 13 16:24:34 2019 +1000 + + [DOCS] copy_to only works one level deep, not recursively (#37249) + +commit a4339ec7e9b6e6cc2ca4511048cae5915ad17b58 +Author: Costin Leau +Date: Sun Jan 13 01:40:22 2019 +0200 + + SQL: Use declared source for error messages (#37161) + + Improve error messages by returning the original SQL statement + declaration instead of trying to reproduce it as the casing and + whitespaces are not preserved accurately leading to small + differences. + + Close #37161 + +commit 03be4dbaca182a360e0a6cf3aa547739f44f1308 +Author: Jason Tedor +Date: Sat Jan 12 14:43:19 2019 -0800 + + Introduce retention lease persistence (#37375) + + This commit introduces the persistence of retention leases by persisting + them in index commits and recovering them when recovering a shard from + store. + +commit 44a1071018b1563a15548fda14f9abbf11985606 +Author: Nhat Nguyen +Date: Sat Jan 12 12:49:48 2019 -0500 + + Make recovery source partially non-blocking (#37291) + + Today a peer-recovery may run into a deadlock if the value of + node_concurrent_recoveries is too high. This happens because the + peer-recovery is executed in a blocking fashion. This commit attempts + to make the recovery source partially non-blocking. I will make three + follow-ups to make it fully non-blocking: (1) send translog operations, + (2) primary relocation, (3) send commit files. + + Relates #36195 + +commit 63fe3c6ed604d2aee3ae5266b35e87f35ff09bee +Author: Armin Braun +Date: Fri Jan 11 23:26:04 2019 +0100 + + Fix PrimaryAllocationIT Race Condition (#37355) + + * Fix PrimaryAllocationIT Race Condition + + * Forcing a stale primary allocation on a green index was tripping the assertion that was removed + * Added a test that this case still errors out correctly + * Made the ability to wipe stopped datanode's data public on the internal test cluster and used it to ensure correct behaviour on the fixed test + * Previously it simply passed because the test finished before the index went green and would NPE when the index was green at the time of the shard store status request, that would then come up empty + * Closes #37345 + +commit 359222c55c50bf9c15062ca16c71420c41d60100 +Author: Marios Trivyzas +Date: Fri Jan 11 23:08:00 2019 +0200 + + SQL: Make `FULL` non-reserved keyword in the grammar (#37377) + + Since `full` can be common as a field name or part of a field name + (e.g.: `full.name` or `name.full`), it's nice if it's not a reserved + keyword of the grammar so a user can use it without resorting to quotes. + + Fixes: #37376 + +commit 85531f0285839ebc2838129cefe8c2b5dc7ea999 +Author: Marios Trivyzas +Date: Fri Jan 11 22:43:17 2019 +0200 + + SQL: [Tests] Fix and enable internalClusterTests (#37300) + + SqlPlugin cannot have more than one public constructor, so for the testing + purposes the `getLicenseState()` should be overriden. + + Fixes: #37191 + + Co-authored-by: Michael Basnight + +commit 5101e518918ffeb05e679b10504a8bb5c5bd2632 +Author: Benjamin Trent +Date: Fri Jan 11 13:29:30 2019 -0600 + + ML: Fix testMigrateConfigs (#37373) + + * ML: :s/execute/get + + * Fixing other broken tests + + * unmuting test + +commit de52ba1f78edced3060804df7301c305b384009c +Author: Zachary Tong +Date: Fri Jan 11 13:16:05 2019 -0500 + + Fix RollupDocumentation test to wait for job to stop + + Also adds some extra state debug information to various log messages + +commit 827ece73c840aab82ef1ab677a83c7c0901ac27e +Author: Gordon Brown +Date: Fri Jan 11 11:11:27 2019 -0700 + + Mute MlConfigMigratorIT.testMigrateConfigs (#37374) + +commit 70cee18e56b638525c83054fe795ca1e8ca4f26a +Author: Nhat Nguyen +Date: Fri Jan 11 13:06:17 2019 -0500 + + Introduce StepListener (#37327) + + This commit introduces StepListener which provides a simple way to write + a flow consisting of multiple asynchronous steps without having nested + callbacks. + + Relates #37291 + +commit 955d3aea194b4709a37f88057b8f2dd9a9af760c +Author: Gordon Brown +Date: Fri Jan 11 09:35:38 2019 -0700 + + Mute testRoundRobinWithFailures (#32190) + +commit 953fb9352f439baa162ee5773e82f253f1e70a0e +Author: David Roberts +Date: Fri Jan 11 16:31:55 2019 +0000 + + [ML] Update error message for process update (#37363) + + When this message was first added the model debug config was + the only thing that could be updated, but now more aspects of + the config can be updated so the message needs to be more + general. + +commit e4391afd98c99e05ce1ca2163d1a096a0405d72b +Author: Martijn van Groningen +Date: Fri Jan 11 17:26:01 2019 +0100 + + Test fix, wait for auto follower to have stopped in the background + + Relates to #36761 + +commit bb6d8784e71857fd5cea7808e219c606c343b6f7 +Author: Christoph Büscher +Date: Fri Jan 11 17:24:12 2019 +0100 + + Switch indices.get rest after backport of `include_type_name` (#37351) + + With the `include_type_name` available now for indices.get on 6.x after the + backport, the corresponsing yaml test can include anything from 6.7 on. + Also changing the RestGetIndicesActionTests base test class. + +commit 093d4a371cca29d6567f6c2b166eff78468beac9 +Author: Boaz Leskes +Date: Fri Jan 11 08:07:35 2019 -0800 + + followup to #37284 with additional feedback + +commit cae71cddfef98ca9768b4798f081ae89e502e02c +Author: Boaz Leskes +Date: Fri Jan 11 07:59:15 2019 -0800 + + Document Seq No powered optimistic concurrency control (#37284) + + Add documentation to describe the new sequence number powered optimistic concurrency control + + Relates #36148 + Relates #10708 + +commit 1eba1d1df91aa1797a7372bd7ceffb7010f869ad +Author: Armin Braun +Date: Fri Jan 11 16:09:26 2019 +0100 + + Fix SnapshotDisruptionIT Race Condition (#37358) + + * Due to a race between retrying the snapshot creation and the failed snapshot create trying to delete the snapshot there is no guarantee that the snapshot is eventually created by retries + * Adjusted the assertion accordingly + * Closes #36779 + +commit f4abf9628aa35138f892db93be5a7a3fb3639a5e +Author: Yannick Welsch +Date: Fri Jan 11 16:06:48 2019 +0100 + + Mock connections more accurately in DisruptableMockTransport (#37296) + + This commit moves DisruptableMockTransport to use a more accurate representation of connection + management, which allows to use the full connection manager and does not require mocking out + any behavior. With this, we can implement restarting nodes in CoordinatorTests. + +commit fad63298082cccaba6c681aec896e7d636998cfe +Author: Jake Landis +Date: Fri Jan 11 08:58:41 2019 -0600 + + add link to freeze from cold phase (#37332) + +commit d21df2a17a2ed96b0f307505ba0c6c66696e05dd +Author: Boaz Leskes +Date: Fri Jan 11 06:23:55 2019 -0800 + + Use Sequence number powered OCC for processing updates (#37308) + + Updates perform realtime get, perform the requested update and then index the document again + using optimistic concurrency control. This PR changes the logic to use sequence numbers instead + of versioning. + + Note that the current versioning logic isn't suffering from the same problem as external OCC + requests because the get and indexing is always done on the same primary. + + Relates #36148 + Relates #10708 + +commit 19a7e0f4eb4e094ddc7f6e1195c685ef22537bb8 +Author: Benjamin Trent +Date: Fri Jan 11 08:03:41 2019 -0600 + + ML: update .ml-state actions to support > 1 index (#37307) + + * ML: Updating .ml-state calls to be able to support > 1 index + + * Matching bulk delete behavior with dbq + + * Adjusting state name + + * refreshing indices before search + + * fixing line length + + * adjusting index expansion options + +commit 1da59db3fbe7099eacce8a022c694ad8b9005aa8 +Author: David Roberts +Date: Fri Jan 11 13:22:35 2019 +0000 + + [ML] Wait for autodetect to be ready in the datafeed (#37349) + + This is a reinforcement of #37227. It turns out that + persistent tasks are not made stale if the node they + were running on is restarted and the master node does + not notice this. The main scenario where this happens + is when minimum master nodes is the same as the number + of nodes in the cluster, so the cluster cannot elect a + master node when any node is restarted. + + When an ML node restarts we need the datafeeds for any + jobs that were running on that node to not just wait + until the jobs are allocated, but to wait for the + autodetect process of the job to start up. In the case + of reassignment of the job persistent task this was + dealt with by the stale status test. But in the case + where a node restarts but its persistent tasks are not + reassigned we need a deeper test. + + Fixes #36810 + +commit 4d3928d4449e0202f48eb5fb15179e4a9e7847e4 +Author: Yannick Welsch +Date: Fri Jan 11 11:25:20 2019 +0100 + + Increase timeouts in UnicastZenPingTests + + Relates to #37268 + +commit ed2b5e80ebec290d8da7fdaa9d3420e64cb1b3c7 +Author: Andrei Stefan +Date: Fri Jan 11 11:24:31 2019 +0200 + + Update discovery-ec2.asciidoc (#28925) + + * Be more clear about the need to defining tags in EC2 to be able to filter + ES instances. + +commit 100ef2cc0ef140cc02bb5b3006e13f7110463f6a +Author: Alpar Torok +Date: Fri Jan 11 11:23:54 2019 +0200 + + Fix artifactId in plugin poms (#37315) + +commit 3a929c7aeac203fdcacbc2470bf6574ca116c97c +Author: Yannick Welsch +Date: Fri Jan 11 10:15:39 2019 +0100 + + Increase assertBusy timeouts for RefreshListenersTests + +commit 82ca2d62decbb496047d1eab4160c98099c19171 +Author: Alpar Torok +Date: Fri Jan 11 10:54:53 2019 +0200 + + Mute CloseWhileRelocatingShardsIT.testCloseWhileRelocatingShards + + Tracked by #37274 + +commit b24dc2c5417af62a007ef0104d8bb6d62488a7b7 +Author: Jim Ferenczi +Date: Fri Jan 11 09:47:46 2019 +0100 + + [TEST] Awaits tasks termination in the RestHighLevelClient tests (#37302) + + This change ensures that TasksIT#testGetValidTask and ReindexIT#testReindexTask + don't leave a non-completed task on the cluster when they finish. + + Closes #35644 + +commit 3e73911cbe81c05d71439fe9230bf4eeacf2e9b6 +Author: Alpar Torok +Date: Fri Jan 11 10:45:40 2019 +0200 + + Mute PrimaryAllocationIT.testForceStaleReplicaToBePromotedToPrimaryOnWrongNode + + Tracking issue: #37345 + +commit 0a50821bb29538431ac80916817012927677c9f7 +Author: Ignacio Vera +Date: Fri Jan 11 09:37:18 2019 +0100 + + Geo: Do not normalize the longitude with value -180 for Lucene shapes (#37299) + + Lucene based shapes should not normalize the longitude value -180 to 180. + +commit 360c430ad7893f083049dbefccb3df9a9000cb81 +Author: Nhat Nguyen +Date: Fri Jan 11 03:33:06 2019 -0500 + + Add runAfter and notifyOnce wrapper to ActionListener (#37331) + + Relates #37291 + +commit 9f3da013d899f458ff56ded1a57fab4818f90e47 +Author: Alexander Reelsen +Date: Fri Jan 11 09:25:05 2019 +0100 + + Date/Time parsing: Use java time API instead of exception handling (#37222) + + * Add benchmark + + * Use java time API instead of exception handling + + when several formatters are used, the existing way of parsing those is + to throw an exception catch it, and try the next one. This is is + considerably slower than the approach taken in joda time, so that + indexing is reduced when a date format like `x||y` is used and y is the + date format being used. + + This commit now uses the java API to parse the date by appending the + date time formatters to each other and does not rely on exception + handling. + + * fix benchmark + + * fix tests by changing formatter, also expose printer + + * restore optional printing logic to fix tests + + * fix tests + + * incorporate review comments + +commit bbd093059f5b18416d46a315b873c473b11648f3 +Author: Alexander Reelsen +Date: Fri Jan 11 09:22:47 2019 +0100 + + Add whitelist to watcher HttpClient (#36817) + + This adds a configurable whitelist to the HTTP client in watcher. By + default every URL is allowed to retain BWC. A dynamically configurable + setting named "xpack.http.whitelist" was added that allows to + configure an array of URLs, which can also contain simple regexes. + + Closes #29937 + +commit 37493c204da0945b7b2cd59f4eec7d4a6e8e1974 +Author: Martijn van Groningen +Date: Fri Jan 11 09:02:46 2019 +0100 + + Unmuted test now that #37239 has been merged and backported. + + Relates to #37231 + +commit 80084138dd64cbdf922838978148a139c64d85df +Author: Ioannis Kakavas +Date: Fri Jan 11 09:22:40 2019 +0200 + + [DOCS] Fix link to role mapping doc + +commit 860a8a7b235a91d6927ac8e2171f039c23b8de3e +Author: Armin Braun +Date: Fri Jan 11 08:07:55 2019 +0100 + + Improve Precision for scaled_float (#37169) + + * Use `toString` and `Bigdecimal` parsing to get intuitive behaviour for `scaled_float` as discussed in #32570 + * Closes #32570 + +commit 822626dadf512e7ff6f31ded00cafeb555cf0729 +Author: Jason Tedor +Date: Thu Jan 10 18:34:55 2019 -0800 + + Make consistent empty retention lease supplier + + This commit makes the use of empty retention lease suppliers to always + be an empty list as opposed to in some cases an empty set. This commit + is solely for consistency reasons, there is no functional change here. + +commit edc95c8a8e1cb0ab890b81894c378ac3f2fd2ab9 +Author: Jason Tedor +Date: Thu Jan 10 18:13:05 2019 -0800 + + Add validation for retention lease construction (#37312) + + This commit adds some simple validation that the values input to the + retention lease constructor our valid values. We will later rely on + these values being within the validated range. + +commit 9fd4d6ba4be313e722f475b6d72e7d2c3ceb2cbd +Author: Jake Landis +Date: Thu Jan 10 17:06:44 2019 -0600 + + remove `+` references for xpack.monitoring.collection.indices (#37282) + + `+` for index name inclusions is no longer supported for 6.x+. This + commit removes references of the `+` from the documenation. System + indices additional example is also included. + + fixes #37237 + +commit b5b93a27464ab2fae2f2bf817bf64cd8102eba9e +Author: Jack Conradson +Date: Thu Jan 10 13:58:00 2019 -0800 + + Rename ParameterMap to DeprecationMap (#37317) + + Mechanical change to rename ParameterMap to DeprecationMap as this + seems more appropriate for an extended Map to issue deprecation warnings. + +commit 434430506b760257f6d3defd4cc5b6be5be8cc2b +Author: markharwood +Date: Thu Jan 10 21:35:19 2019 +0000 + + Type removal - added deprecation warnings to _bulk apis (#36549) + + Added warnings checks to existing tests + Added “defaultTypeIfNull” to DocWriteRequest interface so that Bulk requests can override a null choice of document type with any global custom choice. + Related to #35190 + +commit 04dcb13ac48c12e1f30e99de74908792615f5566 +Author: Lee Hinman +Date: Thu Jan 10 13:38:14 2019 -0700 + + Remove "beta" modifier from ILM documentation (#37326) + +commit e6d3d85db44c07f097facba450f1ee73e87b8640 +Author: Jay Modi +Date: Thu Jan 10 13:27:25 2019 -0700 + + Ensure latch is counted down in ssl reload test (#37313) + + This change ensures we always countdown the latch in the + SSLConfigurationReloaderTests to prevent the suite from timing out in + case of an exception. Additionally, we also increase the logging of the + resource watcher in case an IOException occurs. + + See #36053 + +commit 83f7423cd65c661eeca46fc2f9dd874529ccff53 +Author: Costin Leau +Date: Thu Jan 10 22:18:53 2019 +0200 + + SQL: Fix bug regarding alias fields with dots (#37279) + + Field of types aliases that have dots in name are returned without a + hierarchy by field_caps, as oppose to the mapping api or field with + concrete types, which in turn breaks IndexResolver. + This commit fixes this by creating the backing hierarchy similar to the + mapping api. + + Close #37224 + +commit a2d63ecdc0598a8af9fc3c6ef384a998ef442a7f +Author: Lee Hinman +Date: Thu Jan 10 12:34:44 2019 -0700 + + Link ILM management and policy information in ILM API documentation (#37324) + + Previously these were only linked in a circuitous way rather than being + available from the top level API documentation and "Put Lifecycle" API docs. + This makes them slightly easier to find for a user. + +commit 2d5a8ec59db8e4018615572e7150658b9c37e718 +Author: lcawl +Date: Thu Jan 10 11:10:25 2019 -0800 + + [DOCS] Remove unused screenshots + +commit a57571045ee404468ea5b311033b8b8aae4b89dd +Author: Michael Basnight +Date: Thu Jan 10 12:00:51 2019 -0600 + + Fix rest reindex test for IPv4 addresses (#37310) + + Some of our CI boxes end up giving out an IPv4 address for this + test. This commit allows both v4 and v6 addresses to be used. + +commit b65006e8cd558b5e1a91ce9f185dcda5f6135769 +Author: David Roberts +Date: Thu Jan 10 17:28:00 2019 +0000 + + [ML] Fix ML memory tracker for old jobs (#37311) + + Jobs created in version 6.1 or earlier can have a + null model_memory_limit. If these are parsed from + cluster state following a full cluster restart then + we replace the null with 4096mb to make the meaning + explicit. But if such jobs are streamed from an + old node in a mixed version cluster this does not + happen. Therefore we need to account for the + possibility of a null model_memory_limit in the ML + memory tracker. + +commit a433c4012c51263d49e33cb9c708920f84b86f99 +Author: Julie Tibshirani +Date: Thu Jan 10 09:24:08 2019 -0800 + + Support include_type_name in the field mapping and index template APIs. (#37210) + + * Add include_type_name to the get field mappings API. + * Make sure the API specification lists include_type_name as a boolean. + * Add include_type_name to the get index templates API. + * Add include_type_name to the put index templates API. + +commit 9de62f126278d1866e381418dec4af1aea7cd0b4 +Author: Tim Brooks +Date: Thu Jan 10 09:17:20 2019 -0700 + + Increase IO direct byte buffers to 256KB (#37283) + + Currently we read and write 64KB at a time in the nio libraries. As a + single byte buffer per event loop thread does not consume much memory, + there is little reason to not increase it further. This commit increases + the buffer to 256KB but still limits a single write to 64KB. The write + limit could be increased, but too high of a write limit will lead to + copying more data (if all the data is not flushed and needs to be copied + on the next call). This is something to explore in the future. + +commit 71633775fd1a3cdad497233848a83ccd42020602 +Author: Jay Modi +Date: Thu Jan 10 09:06:16 2019 -0700 + + Security: reorder realms based on last success (#36878) + + This commit reorders the realm list for iteration based on the last + successful authentication for the given principal. This is an + optimization to prevent unnecessary iteration over realms if we can + make a smart guess on which realm to try first. + +commit fcf7df3edaad3b346ddfd5b536d8a410996663cf +Author: Ryan Ernst +Date: Thu Jan 10 07:44:40 2019 -0800 + + Core: Handle security manager permission for deprecation log rolling (#37281) + + When the deprecation log is written to within scripting support code + like ScriptDocValues, it runs under the reduces privileges of scripts. + Sometimes this can trigger log rolling, which then causes uncaught + security errors, as was handled in #28485. While doing individual + deprecation handling within each deprecation scripting location is + possible, there are a growing number of deprecations in scripts. + + This commit wraps the logging call within the deprecation logger use a + doPrivileged block, just was we would within individual logging call + sites for scripting utilities. + +commit 46237faa97978327be138fceeadfb66329c799ff +Author: Armin Braun +Date: Thu Jan 10 16:28:38 2019 +0100 + + Fail Stale Primary Alloc. Req. without Data (#37226) + + * Get indices shard store status before enqueuing the reallocation state update task to prevent + tasks that would fail because a node does not hold a stale copy of the shard on a best effort basis + * Closes #37098 + +commit 6d81e7c3e7100ddbef1ee812093f4386395e8f2b +Author: Martijn van Groningen +Date: Thu Jan 10 15:54:34 2019 +0100 + + [CCR] FollowingEngine should fail with 403 if operation has no seqno assigned (#37213) + + Fail with a 403 when indexing a document directly into a follower index. + + In order to test this change, I had to move specific assertions into a dedicated class and + disable assertions for that class in the rest qa module. I think that is the right trade off. + +commit c812e6aea6a66092188d56090ff5a77a48a81a8b +Author: Przemyslaw Gomulka +Date: Thu Jan 10 15:23:34 2019 +0100 + + Fix line length in org.elasticsearch.routing (#37253) + + Remove the line length suppression for this package and fix offending + lines + + relates: #34884 + +commit df488720e02ad4bf7398cf35a98a5abd171ecad0 +Author: Martijn van Groningen +Date: Thu Jan 10 15:02:30 2019 +0100 + + [CCR] Make shard follow tasks more resilient for restarts (#37239) + + If a running shard follow task needs to be restarted and + the remote connection seeds have changed then + a shard follow task currently fails with a fatal error. + + The change creates the remote client lazily and adjusts + the errors a shard follow task should retry. + + This issue was found in test failures in the recently added + ccr rolling upgrade tests. The reason why this issue occurs + more frequently in the rolling upgrade test is because ccr + is setup in local mode (so remote connection seed will become stale) and + all nodes are restarted, which forces the shard follow tasks to get + restarted at some point during the test. Note that these tests + cannot be enabled yet, because this change will need to be backported + to 6.x first. (otherwise the issue still occurs on non upgraded nodes) + + I also changed the RestartIndexFollowingIT to setup remote cluster + via persistent settings and to also restart the leader cluster. This + way what happens during the ccr rolling upgrade qa tests, also happens + in this test. + + Relates to #37231 + +commit 26cb7466ef93c8b915741a379cb614a3600a5226 +Author: Armin Braun +Date: Thu Jan 10 14:13:20 2019 +0100 + + SNAPSHOT+TESTS: Stabilize SnapshotDisruptionIT (#37289) + + * Ensure retry by busy assert on SnapshotMissingException + * Closes #36739 + +commit 44acb016a693221d77687851db9ba764a1ea71be +Author: Martijn van Groningen +Date: Thu Jan 10 14:04:55 2019 +0100 + + [HLRC] Ignore unknown fields in responses of CCR APIs (#36821) + + Otherwise hlrc fails if new fields are introduced in ccr api responses + in future versions. + +commit 61b54196c4d2a79ea9fa416cf436cac053c3c271 +Author: Luca Cavanna +Date: Thu Jan 10 13:38:39 2019 +0100 + + [TEST] Fixed compile issue in SnapshotsServiceTests + + Relates to #37203 + +commit 71287b0759d4a1562206446c6196bcec6cae663a +Author: Alexander Reelsen +Date: Thu Jan 10 13:20:24 2019 +0100 + + Remove unused EpochMillisDateFormatter (#37293) + + This class has been superceded by a custom java time epoch millis date + parser. + +commit d499233068a840a632b42b5f890708241666912f +Author: Yannick Welsch +Date: Thu Jan 10 12:57:50 2019 +0100 + + Zen2: Add join validation (#37203) + + Adds join validation to Zen2, which prevents a node from joining a cluster when the node does not + have the right ES version or does not satisfy any other of the join validation constraints. + +commit cd608848e7fb6645816df9044b0b6419f16dfe5a +Author: Christoph Büscher +Date: Thu Jan 10 12:50:07 2019 +0100 + + Remove deprecated QUERY_AND_FETCH SearchType (#37257) + + This SearchType was deprecated since at least 6.0 and according to the + documentation is only kept around for pre-5.3 requests. Removing and leaving a + comment as placeholder so we don't reuse the byte value associated with it + without further consideration. + +commit 3d66764660debae71a059136a36ea5412c905c2c +Author: Alpar Torok +Date: Wed Jan 9 20:49:42 2019 +0200 + + Mute watcher SingleNodeTests + + Tracking: #36782 + +commit 1a41d84536aa6eca84d6c929af595c91edf3a525 +Author: Martijn van Groningen +Date: Thu Jan 10 09:48:26 2019 +0100 + + [CCR] Resume follow Api should not require a request body (#37217) + + Closes #37022 + +commit eb12de550aeae0a4d88927183c7c8f3af8facf31 +Author: Alexander Reelsen +Date: Thu Jan 10 09:26:01 2019 +0100 + + Java Time: Fix timezone parsing (#37262) + + * Java Time: Fix timezone parsing + + An independent test uncovered an issue when parsing a timezone + containing a colon like `01:00` - some formats did not properly support + this. + + This commit adds test for all formats in the dueling tests and fixes a + few issues with existing date formatters. + + * fix tests, so they run under java8 + +commit b2e8437424660a7552f638a7749dc652e606ae49 +Author: Alexander Reelsen +Date: Thu Jan 10 09:25:36 2019 +0100 + + Tests: Add ElasticsearchAssertions.awaitLatch method (#36777) + + * Tests: Add ElasticsearchAssertions.awaitLatch method + + Some tests are using assertTrue(latch.await(...)) in their code. This + leads to an assertion error without any error message. This adds a + method which has a nicer error message and can be used in tests. + + * fix forbidden apis + + * fix spaces + +commit 4a92de214a0811bfde65311aae65979866b0db5e +Author: Andrei Stefan +Date: Thu Jan 10 09:51:51 2019 +0200 + + SQL: Proper handling of COUNT(field_name) and COUNT(DISTINCT field_name) (#37254) + + * provide overriden `hashCode` and toString methods to account for `DISTINCT` + * change the analyzer for scenarios where `COUNT ` and `COUNT DISTINCT` have different paths + * defined a new `filter` aggregation encapsulating an `exists` query to filter out null or missing values + +commit d625b79df2b1bb3a950038dfd39d27e93d4b57b0 +Author: Michael Basnight +Date: Wed Jan 9 22:09:34 2019 -0600 + + Add getZone to JodaCompatibleZonedDateTime (#37084) + + The ZonedDateTime#getZone() was not accessible via the Joda shim. This + commit adds getZone() and exposes it through painless. + +commit 586093ec5e96bf657bc89040b0b6c822fd4f4678 +Author: Jim Ferenczi +Date: Thu Jan 10 00:26:35 2019 +0100 + + Handle TopFieldDocs copy in TopDocsCollectorContext + + This commit fixes the clone of TopFieldDocs. + + Relates #37179 + Relates #37266 + +commit 234059d2c091f192a1be6393c60f2a10e902dc54 +Author: Simon Willnauer +Date: Wed Jan 9 23:46:31 2019 +0100 + + Enable Bulk-Merge if all source remains (#37269) + + Today we still wrap recovery source readers on merge even if we + keep all documents recovery source. This basically disables bulk + merging for stored fields. This change skips wrapping if all docs + sources are kept anyway. + +commit 95479f17667c7d27fc480c260a1c380d681d8b93 +Author: Jim Ferenczi +Date: Wed Jan 9 22:57:34 2019 +0100 + + Ensure that a non static top docs is created during the search phase + + This change fixes an unreleased bug that trips an assertion because a static instance + shared among threads is modified during the search. This commit copies the static + instance in order to ensure that each thread can modify the value without modifying + the other instances. + + Closes #37179 + Closes #37266 + +commit 195873002bd3724c839a4a211c3aa1428f104a3a +Author: Jake Landis +Date: Wed Jan 9 14:47:47 2019 -0600 + + ingest: compile mustache template only if field includes '{{'' (#37207) + + * ingest: compile mustache template only if field includes '{{'' + + Prior to this change, any field in an ingest node processor that supports + script templates would be compiled as mustache template regardless if they + contain a template or not. Compiling normal text as mustache templates is + harmless. However, each compilation counts against the script compilation + circuit breaker. A large number of processors without any templates or scripts + could un-intuitively trip the too many script compilations circuit breaker. + + This change simple checks for '{{' in the text before it attempts to compile. + + fixes #37120 + +commit df3b58cb04c0443142221884380d67d86d7da6e0 +Author: Benjamin Trent +Date: Wed Jan 9 14:25:35 2019 -0600 + + ML: add migrate anomalies assistant (#36643) + + * ML: add migrate anomalies assistant + + * adjusting failure handling for reindex + + * Fixing request and tests + + * Adding tests to blacklist + + * adjusting test + + * test fix: posting data directly to the job instead of relying on datafeed + + * adjusting API usage + + * adding Todos and adjusting endpoint + + * Adding types to reindexRequest + + * removing unreliable "live" data test + + * adding index refresh to test + + * adding index refresh to test + + * adding index refresh to yaml test + + * fixing bad exists call + + * removing todo + + * Addressing remove comments + + * Adjusting rest endpoint name + + * making service have its own logger + + * adjusting validity check for newindex names + + * fixing typos + + * fixing renaming + +commit c71060fa0118ac9de2b292fe0c2ef587e0cb35ce +Author: jaymode +Date: Wed Jan 9 12:17:43 2019 -0700 + + Test: fix race in auth result propagation test + + This commit fixes a race condition in a test introduced by #36900 that + verifies concurrent authentications get a result propagated from the + first thread that attempts to authenticate. Previously, a thread may + be in a state where it had not attempted to authenticate when the first + thread that authenticates finishes the authentication, which would + cause the test to fail as there would be an additional authentication + attempt. This change adds additional latches to ensure all threads have + attempted to authenticate before a result gets returned in the + thread that is performing authentication. + +commit 722b850efd1cd269233bfba22d2897d0749d8aee +Author: Ryan Ernst +Date: Wed Jan 9 10:57:26 2019 -0800 + + Build: Make assemble a noop in bwc projects (#37245) + + This commit makes the assemble tasks in the bwc projects noops by + setting the dependsOn directly. While we can not remove things from + dependsOn, we can still completely override the dependencies. + + closes #33581 + +commit 29c895b55c4d52fa5649fcfd122f97c58113cb93 +Author: Ryan Ernst +Date: Wed Jan 9 10:56:30 2019 -0800 + + Build: Quiet naming convention logging (#37244) + + This commit moves log statements related to classification of naming + convention checks for tests to debug level. At info level they emit an + enormous amount of output in CI, while these are not generally useful + for debugging normal build failures. + +commit 95eef77ad4eacb90d20ffc140a1c030eb5d354b6 +Author: Jack Conradson +Date: Wed Jan 9 10:55:52 2019 -0800 + + [Style] Fix line length violations for threadpool, indexing, and script packages (#37205) + +commit cfa58a51afd1cd41d8aebb024d6f8987373b21d1 +Author: Tim Brooks +Date: Wed Jan 9 11:46:24 2019 -0700 + + Add TLS/SSL channel close timeouts (#37246) + + Closing a channel using TLS/SSL requires reading and writing a + CLOSE_NOTIFY message (for pre-1.3 TLS versions). Many implementations do + not actually send the CLOSE_NOTIFY message, which means we are depending + on the TCP close from the other side to ensure channels are closed. In + case there is an issue with this, we need a timeout. This commit adds a + timeout to the channel close process for TLS secured channels. + + As part of this change, we need a timer service. We could use the + generic Elasticsearch timeout threadpool. However, it would be nice to + have a local to the nio event loop timer service dedicated to network needs. In + the future this service could support read timeouts, connect timeouts, + request timeouts, etc. This commit adds a basic priority queue backed + service. Since our timeout volume (channel closes) is very low, this + should be fine. However, this can be updated to something more efficient + in the future if needed (timer wheel). Everything being local to the event loop + thread makes the logic simple as no locking or synchronization is necessary. + +commit 13b8bad2b86c92510a05bd0be1e35c510769d1c2 +Author: Morris Schreibman +Date: Wed Jan 9 19:12:31 2019 +0200 + + [DOCS] Clarify client settings (#31469) + +commit 6a5f3f05f405095ef97c6e40eca4cbebcc4bb82f +Author: Alpar Torok +Date: Wed Jan 9 19:27:01 2019 +0200 + + Fix build on Fips + + testing convetions need to be disabled if the test task is for fips. + +commit d68d6aa00cb64f74485b69e081d7432fa887b358 +Author: Alpar Torok +Date: Wed Jan 9 18:26:08 2019 +0200 + + Compatability mode for forbidden APIs for Java 12 (#37199) + +commit 4efbda1bc0d13ada5f0706fd6a5897356acc4506 +Author: Albert Zaharovits +Date: Wed Jan 9 18:14:22 2019 +0200 + + [DOCS] Rolling upgrade with old internal indices (#37184) + + Upgrading the Elastic Stack perfectly documents the process to + upgrade ES from 5 to 6 when internal indices are present. However, + the rolling upgrade docs do not mention anything about internal indices. + + This adds a warning in the rolling upgrade procedure, highlighting that + internal indices should be upgraded before the rolling upgrade procedure + can be started. + +commit 85a603ee6192055332e7b002b2a170a38bb48774 +Author: Evangelos Chatzikalymnios +Date: Wed Jan 9 17:01:39 2019 +0100 + + Use List instead of priority queue for stable sorting in bucket sort aggregator (#36748) + + Update BucketSortPipelineAggregator to use a List and Collections.sort() for sorting instead of a priority queue. This preserves the order for equal values. Closes #36322. + +commit eacc63b032cb0cb055c41830e2d2eb29e61be030 +Author: Armin Braun +Date: Wed Jan 9 16:53:49 2019 +0100 + + TESTS: Real Coordinator in SnapshotServiceTests (#37162) + + * TESTS: Real Coordinator in SnapshotServiceTests + + * Introduce real coordinator in SnapshotServiceTests to be able to test network disruptions realistically + * Make adjustments to cluster applier service so that we can pass a mocked single threaded executor for tests + +commit ae086ebcc4467eb58e14ed3fcea4d02b382a37be +Author: Alpar Torok +Date: Wed Jan 9 16:55:11 2019 +0200 + + Muting SnapshotDisruptionIT + + Tracked in #36779 + +commit 7de4d2cb0fb448c2dd4e5d60efc8281e1d5aa92f +Author: Alpar Torok +Date: Wed Jan 9 16:43:24 2019 +0200 + + Mute failing test ChildQuerySearchIT + + Tracked in #37266 + +commit c149bb8cc2544c02997358de762c231c48040704 +Author: Christoph Büscher +Date: Wed Jan 9 14:17:17 2019 +0100 + + Support 'include_type_name' in RestGetIndicesAction (#37149) + + This change adds support for the 'include_type_name' parameter for the + indices.get API. This parameter, which defaults to `false` starting in 7.0, + changes the response to not include the indices type names any longer. + + If the parameter is set in the request, we additionally emit a deprecation + warning since using the parameter should be only temporarily necessary while + adapting to the new response format and we will remove it with the next major + version. + +commit 84d520b0e59099ce7bff2714bf82773554e55f69 +Author: Jim Ferenczi +Date: Wed Jan 9 13:35:41 2019 +0100 + + Throw an ISE rather than an hard assertion in SearchPhaseController#getTotalHits + + This change turns an assertion into an IllegalStateException in SearchPhaseController#getTotalHits. + The goal is to help identify the cause of the failures in https://github.com/elastic/elasticsearch/issues/37179 + which seems to fail only in CI. + The assertion will be restored when the issue is solved (NORELEASE). + +commit dc371ef59352ac7d2d5e59ad393f817bf44e2fd0 +Author: Tanguy Leroux +Date: Wed Jan 9 13:34:38 2019 +0100 + + [Tests] Fix ReopenWhileClosingIT with correct min num shards + + The test intercepts TransportVerifyShardBeforeCloseAction shard + requests, so it needs a minimum of 2 primary shards on 2 different + nodes to correctly intercepts requests. + +commit 7ff846ffb24a51f27676a4f1ea39cde1af272318 +Author: Alpar Torok +Date: Wed Jan 9 13:43:21 2019 +0200 + + mute failing test: org.elasticsearch.index.shard.RefreshListenersTests.testConcurrentRefresh + +commit 91225853591a8e35eaecdee4a67fde673d83052b +Author: Martijn van Groningen +Date: Wed Jan 9 12:17:47 2019 +0100 + + [CCR] Added more logging. + +commit f1f5d834c308ebb900be932a2d839e1b833ba3f5 +Merge: e0ce73713f7 096a83183e3 +Author: Tanguy Leroux +Date: Wed Jan 9 11:48:57 2019 +0100 + + Merge branch 'close-index-api-refactoring' + +commit e0ce73713f7292954ff66691919abf62d71442cd +Author: David Roberts +Date: Wed Jan 9 10:42:47 2019 +0000 + + [ML] Stop datafeeds running when their jobs are stale (#37227) + + We already had logic to stop datafeeds running against + jobs that were OPENING, but a job that relocates from + one node to another while OPENED stays OPENED, and this + could cause the datafeed to fail when it sent data to + the OPENED job on its new node before it had a + corresponding autodetect process. + + This change extends the check to stop datafeeds running + when their job is OPENING _or_ stale (i.e. has not had + its status reset since relocating to a different node). + + Relates #36810 + +commit 096a83183e3eac6da889bc9d56de5ed881ff9818 +Merge: 7f6fe14b668 f14cff21027 +Author: Tanguy Leroux +Date: Wed Jan 9 10:52:46 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit f14cff2102704b627b18a641f38000ef319e746b +Author: David Roberts +Date: Wed Jan 9 08:51:00 2019 +0000 + + [TEST] Ensure interrupted flag reset after test that sets it (#37230) + + Test fix to stop a problem in one test leaking into a different + test and causing that other test to spuriously fail. + +commit 224475e6aa65828e94912803b4aa066de5c1f7ed +Author: Armin Braun +Date: Wed Jan 9 09:45:37 2019 +0100 + + MINOR: Some Cleanups in Snapshots (#37175) + + * Removed pre v6 Bwc logic + comments + * Removed some dead code + +commit 9040a96daf5e052556ec7140d3fbb03f1ece1b4f +Author: Yannick Welsch +Date: Wed Jan 9 09:32:24 2019 +0100 + + Zen2: Add test that checks that CoordinatorTests are reproducible (#37225) + + Ensures reproducibility of the tests in CoordinatorTests. + +commit 7f6fe14b668baff79725972cb4bfe28661e50586 (es/close-index-api-refactoring) +Merge: 6e852dfa7c9 0a40564068d +Author: Tanguy Leroux +Date: Wed Jan 9 09:26:05 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 0a40564068d024fd97051c5839c86f92a95b357f +Author: Tanguy Leroux +Date: Wed Jan 9 09:24:56 2019 +0100 + + [Tests] Change cluster scope in CorruptedFileIT and FlushIT (#37229) + + These tests failed on CI multiple times in the past weeks because they use a + test cluster with a SUITE scope that recreates nodes between tests. With such + a scope, nodes can be recreated in between test executions and can inherit a + node id from a previous test execution, while they are assigned a random data + path. With the successive node recreations it is possible that a newly recreated + node shares the same node id (but different data path) as a non recreated node. + + This commit changes the cluster scope of the CorruptedFileIT and FlushIT + tests which often fail. + + The failure is reproducable with : + + ./gradlew :server:integTest -Dtests.seed=EF3A50C225CF377 + -Dtests.class=org.elasticsearch.index.store.CorruptedFileIT + -Dtests.security.manager=true -Dtests.locale=th-TH-u-nu-thai-x-lvariant-TH -Dtests.timezone=America/Rio_Branco + -Dcompiler.java=11 -Druntime.java=8 + +commit d9e2ebca674c8834d83594deb24fdced4f720061 +Author: David Turner +Date: Wed Jan 9 08:18:25 2019 +0000 + + Add more detail to recovery bandwidth limit docs (#37156) + +commit 9049263c2c08f4e5a02188d36997049b7668649d +Author: Ioannis Kakavas +Date: Wed Jan 9 07:29:22 2019 +0200 + + Handle malformed license signatures (#37137) + + This commit adds a more user friendly error message when a license + signature is malformed/truncated in a way that it cannot be + meaningfully parsed. + +commit 2a79c468f8624c3ccb452884dfd99e639d2d47da +Author: Ioannis Kakavas +Date: Wed Jan 9 07:19:34 2019 +0200 + + Ensure that ActionListener is called exactly once + + This bug was introduced in #36893 and had the effect that + execution would continue after calling onFailure on the the + listener in checkIfTokenIsValid in the case that the token is + expired. In a case of many consecutive requests this could lead to + the unwelcome side effect of an expired access token producing a + successful authentication response. + +commit 38b698d4554d196ecc4f6fcfb52ae5bdeb24ff5b +Author: Jun Ohtani +Date: Wed Jan 9 12:42:00 2019 +0900 + + [Analysis] Deprecate Standard Html Strip Analyzer in master (#26719) + + * [Analysis] Deprecate Standard Html Strip Analyzer + + Deprecate only Standard Html Strip Analyzer + If user create index with the analyzer since 7.0, es throws an exception. + If an index was created before 7.0, es issue deprecation log + We will remove it in 8.0 + + Related #4704 + +commit ec32e66088c0001b0d3475ef06223de51d48b999 +Author: Mayya Sharipova +Date: Tue Jan 8 21:46:41 2019 -0500 + + Deprecate reference to _type in lookup queries (#37016) + + Relates to #35190 + +commit 5f2fbedd8c7570bcc2ea74f38dce0b1736a85cee +Author: Marios Trivyzas +Date: Tue Jan 8 23:56:00 2019 +0200 + + SQL: Replace String.format() with LoggerMessageFormat.format() (#37216) + + Fixes: #36532 + +commit 87ac3103f426e4ab5c66078d5d41166885ef4100 +Author: Nhat Nguyen +Date: Tue Jan 8 16:12:18 2019 -0500 + + Do not mutate RecoveryResponse (#37204) + + Today we create a global instance of RecoveryResponse then mutate it + when executing each recovery step. This is okay for the current + sequential recovery flow but not suitable for an asynchronous recovery + which we are targeting. With this commit, we return the result of each + step separately, then construct a RecoveryResponse at the end. + + Relates #37174 + +commit 86b71930f6a01bf33171651ac951a70626f8a835 (es/backup_security_index) +Author: Ryan Ernst +Date: Tue Jan 8 11:04:54 2019 -0800 + + Remove more checkstyle suppressions (#36735) + + This commit removes several more line length violations from checkstyle + suppression. + + relates #34884 + +commit d6608caf55af64dc74eeaf2f1aa67b6ef9ca34bf +Author: Martijn van Groningen +Date: Tue Jan 8 16:51:52 2019 +0100 + + Muted rolling upgrade tests. + + Relates to #37231 + +commit 1514bbcdde36d2f64c4391ec6874e7d0aa75ab5f +Author: Jay Modi +Date: Tue Jan 8 08:52:12 2019 -0700 + + Security: propagate auth result to listeners (#36900) + + After #30794, our caching realms limit each principal to a single auth + attempt at a time. This prevents hammering of external servers but can + cause a significant performance hit when requests need to go through a + realm that takes a long time to attempt to authenticate in order to get + to the realm that actually authenticates. In order to address this, + this change will propagate failed results to listeners if they use the + same set of credentials that the authentication attempt used. This does + prevent these stalled requests from retrying the authentication attempt + but the implementation does allow for new requests to retry the + attempt. + +commit dd69553d4d6d57b64664349067b839aca585bbca +Author: Michael Basnight +Date: Tue Jan 8 09:11:58 2019 -0600 + + HLRC: Use nonblocking entity for requests (#32249) + + Previously the HLRC used a blocking ByteArrayEntity, but the Request + class also allows to set a NByteArrayEntity, and defaults to nonblocking + when calling the createJsonEntity method. This commit cleans up all the + uses of ByteArrayEntity in the RequestConverters to use the nonblocking + entity. + +commit 054c3bb04f281cdc71a98e2ac138c4e908943924 +Author: Jim Ferenczi +Date: Tue Jan 8 13:31:53 2019 +0100 + + Fix topDocs.totalHits assignment on scroll queries (#37180) + + This change fixes an unreleased bug that assigns the wrong totalHits to scroll + queries. + + Closes #37179 + +commit 20c2c439e7eaaf7faa8e6e6ae4af1e1316037523 +Author: Alpar Torok +Date: Tue Jan 8 13:43:10 2019 +0200 + + Upgrade to Gradle 5.1 (#36767) + +commit 6344e9a3ce485d6deafb78082dc5f757788e83ec +Author: Alpar Torok +Date: Tue Jan 8 13:39:03 2019 +0200 + + Testing conventions: add support for checking base classes (#36650) + +commit 0c68eead519cc09c8f9ef657f550379336d5b326 +Author: Alpar Torok +Date: Tue Jan 8 13:00:55 2019 +0200 + + Mute failing test clusters test + + Tracking issue: #37218 + +commit 6e852dfa7c952f84bca2fb8b785225db428c1e49 +Merge: 6c78aff8303 c980cc12df5 +Author: Tanguy Leroux +Date: Tue Jan 8 11:28:51 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit c980cc12df569651ff2c83a6e61c4a4ae1216132 +Author: Martijn van Groningen +Date: Tue Jan 8 11:05:18 2019 +0100 + + Added CCR rolling upgrade tests (#36648) + + Added CCR rolling upgrade tests. + +commit 1d371a427faaf5c6ed066d7592dc0fa912400669 +Author: Andrew Banchich +Date: Tue Jan 8 04:22:50 2019 -0500 + + [Docs] Fix wrong math in overview.asciidoc (#37209) + +commit 6c78aff830379dbaa11869919f6ade9b84fdff4b +Merge: d70ebfd1d6f 67f97e36861 +Author: Tanguy Leroux +Date: Tue Jan 8 10:00:47 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 67f97e36861ce3e20c14735f26e2c6d09e18792c +Author: Luca Cavanna +Date: Tue Jan 8 09:53:20 2019 +0100 + + Re-enable bwc tests + + Relates to #37187 + +commit d70ebfd1d6f06c9f743e7c4eb9ce893980895a9c +Merge: 97bf4d71763 3fad9d25f62 +Author: Tanguy Leroux +Date: Tue Jan 8 09:17:48 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 3fad9d25f622be5857854c2cb677426393172871 +Author: Andrei Stefan +Date: Tue Jan 8 08:47:35 2019 +0200 + + SQL: fix COUNT DISTINCT filtering (#37176) + + * Use `_count` aggregation value only for not-DISTINCT COUNT function calls + * COUNT DISTINCT will use the _exact_ version of a field (the `keyword` sub-field for example), if there is one + +commit b0665963e8b4f6ab1264ffc1168d0a20adb81ea2 +Author: Tim Brooks +Date: Mon Jan 7 23:35:22 2019 -0700 + + Make remote ping and compress settings dynamic (#37200) + + Traditionally remote clusters can be configured dynamically. However, + the compress and ping settings are not currently set to be configured + dynamically. This commit changes that. + +commit c8c596cead495c7cd7c1834ef5a871e4bf2d6251 +Author: Jason Tedor +Date: Mon Jan 7 22:03:52 2019 -0800 + + Introduce retention lease expiration (#37195) + + This commit implements a straightforward approach to retention lease + expiration. Namely, we inspect which leases are expired when obtaining + the current leases through the replication tracker. At that moment, we + clean the map that persists the retention leases in memory. + +commit c5aac4705d7e5afff701830f1a9e1e82765db6eb +Author: Julie Tibshirani +Date: Mon Jan 7 17:56:40 2019 -0800 + + Revert "Stop automatically nesting mappings in index creation requests. (#36924)" + + This reverts commit ac1c6940d20b9ac993422b279d8f3a8bd73a3527. + +commit 55d3ca3aa853e50fc72d6fbe701881540818af51 +Author: Ryan Ernst +Date: Mon Jan 7 16:15:30 2019 -0800 + + Core: Rework epoch time parsing for java time (#36914) + + This commit converts the epoch time parsing implementation which uses + the java time api to create DateTimeFormatters instead of DateFormatter + implementations. This will allow multi formats for java time to be + implemented in a single DateTimeFormatter in a future change. + +commit 56e472bfbcf6b16e2fb518d9b183d7dfcaea476d +Author: Christoph Büscher +Date: Tue Jan 8 00:14:50 2019 +0100 + + Removing unused methods in Numbers (#37186) + + Remove several unused helper methods. Most of them are one-liners and should + be easier to be used from the corresponding primitive wrapper classes. + The bytes array conversion methods are unused as well, it should be easy to + re-create them if needed. + +commit 3b48b9986110a3dbb89d3dbf5ad35f9e8b98ef00 +Author: Jason Tedor +Date: Mon Jan 7 14:59:24 2019 -0800 + + Fix setting by time unit (#37192) + + This commit fixes an issue with a settings builder method that allows + setting a duration by time unit. In particular, this method can suffer + from a loss of precision. For example, if the input duration is 1500 + microseconds then internally we are converting this to "1ms", + demonstrating the loss of precision. Instead, we should internally + convert this to a TimeValue that correctly represents the input + duration, and then convert this to a string using a method that does not + lose the unit. That is what this commit does. + +commit 382e4d39efa560fed16a2b5f3d98416920c70e8d +Author: lcawl +Date: Mon Jan 7 14:32:36 2019 -0800 + + [DOCS] Cleans up xpackml attributes + +commit 6b376a1ff4d6ea49f9fa42f828627e5a0444f51c +Author: Benjamin Trent +Date: Mon Jan 7 15:18:38 2019 -0600 + + ML: fix delayed data annotations on secured cluster (#37193) + + * changing executing context for writing annotation + + * adjusting user + + * removing unused import + +commit e34658edba68ae4bbc9c05562322fee073a28963 +Author: Luca Cavanna +Date: Mon Jan 7 11:38:57 2019 +0100 + + Update version in SearchRequest and related test + + Version needs to be updated after backporting #36997 & #37142 + where we added support for providing and serializing localClusterAlias + as well ass absoluteStartMillis. + + Relates to #36997 & #37142 + +commit 97bf4d717636c1fa77a2bcee2982f086a78e06b7 +Merge: e149b0852e2 f307847f29b +Author: Tanguy Leroux +Date: Mon Jan 7 18:38:27 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit f307847f29beeaabdfb6eb2aff486c881dbeca06 +Author: Lisa Cawley +Date: Mon Jan 7 09:11:14 2019 -0800 + + [DOCS] Adds overview and API ref for cluster voting configurations (#36954) + +commit 1780ced82dc96446a5988b8e3462c50261ef9cc1 +Author: Benjamin Trent +Date: Mon Jan 7 10:58:55 2019 -0600 + + ML: changing JobResultsProvider.getForecastRequestStats to support > 1 index (#37157) + + * ML: changing JobResultsProvider.getForecastRequestStats to support more than one index + + * moving to use idsQuery() + +commit 9602d794c62ea526ef2ab5f610e746763d78bdd1 +Author: Christophe Bismuth +Date: Mon Jan 7 17:12:58 2019 +0100 + + Separate out validation of groups of settings (#34184) + + Today, a setting can declare that its validity depends on the values of other + related settings. However, the validity of a setting is not always checked + against the correct values of its dependent settings because those settings' + correct values may not be available when the validator runs. + + This commit separates the validation of a settings updates into two phases, + with separate methods on the `Setting.Validator` interface. In the first phase + the setting's validity is checked in isolation, and in the second phase it is + checked again against the values of its related settings. Most settings only + use the first phase, and only the few settings with dependencies make use of + the second phase. + +commit 9d0e0eb0f3c13404d862795802aa6f0ee7e756b8 +Author: David Turner +Date: Mon Jan 7 16:05:00 2019 +0000 + + [Zen2] Remove initial master node count setting (#37150) + + The `cluster.unsafe_initial_master_node_count` setting was introduced as a + temporary measure while the design of `cluster.initial_master_nodes` was being + finalised. This commit removes this temporary setting, replacing it with usages + of `cluster.initial_master_nodes` where appropriate. + +commit e149b0852e206574faad54ab08f8a61827f73e8d +Author: Tanguy Leroux +Date: Mon Jan 7 16:44:59 2019 +0100 + + [Close Index API] Add unique UUID to ClusterBlock (#36775) + + This commit adds a unique id to cluster blocks, so that they can be uniquely + identified if needed. This is important for the Close Index API where multiple + concurrent closing requests can be executed at the same time. By adding a + UUID to the cluster block, we can generate unique "closing block" that can + later be verified on shards and then checked again from the cluster state + before closing the index. When the verification on shard is done, the closing + block is replaced by the regular INDEX_CLOSED_BLOCK instance. + + If something goes wrong, calling the Open Index API will remove the block. + + Related to #33888 + +commit c0f8c89172a1bbfa81ce5201c035293af8b4e110 +Author: Jason Tedor +Date: Mon Jan 7 07:43:57 2019 -0800 + + Introduce shard history retention leases (#37167) + + This commit is the first in a series which will culminate with + fully-functional shard history retention leases. + + Shard history retention leases are aimed at preventing shard history + consumers from having to fallback to expensive file copy operations if + shard history is not available from a certain point. These consumers + include following indices in cross-cluster replication, and local shard + recoveries. A future consumer will be the changes API. + + Further, index lifecycle management requires coordinating with some of + these consumers otherwise it could remove the source before all + consumers have finished reading all operations. The notion of shard + history retention leases that we are introducing here will also be used + to address this problem. + + Shard history retention leases are a property of the replication group + managed under the authority of the primary. A shard history retention + lease is a combination of an identifier, a retaining sequence number, a + timestamp indicating when the lease was acquired or renewed, and a + string indicating the source of the lease. Being leases they have a + limited lifespan that will expire if not renewed. The idea of these + leases is that all operations above the minimum of all retaining + sequence numbers will be retained during merges (which would otherwise + clear away operations that are soft deleted). These leases will be + periodically persisted to Lucene and restored during recovery, and + broadcast to replicas under certain circumstances. + + This commit is merely putting the basics in place. This first commit + only introduces the concept and integrates their use with the soft + delete retention policy. We add some tests to demonstrate the basic + management is correct, and that the soft delete policy is correctly + influenced by the existence of any retention leases. We make no effort + in this commit to implement any of the following: + - timestamps + - expiration + - persistence to and recovery from Lucene + - handoff during primary relocation + - sharing retention leases with replicas + - exposing leases in shard-level statistics + - integration with cross-cluster replication + + These will occur individually in follow-up commits. + +commit a7c3d5842a5d6448fcf71d436ceedf823f58d5c4 +Author: Alpar Torok +Date: Mon Jan 7 17:24:19 2019 +0200 + + Split third party audit exclusions by type (#36763) + +commit a233db7367844e4c91fc671a9dbb2bc0f8ba8dfe +Author: Jason Tedor +Date: Mon Jan 7 07:13:50 2019 -0800 + + Fix handling of fractional byte size value settings (#37172) + + This commit addresses an issue when setting a byte size value setting + using a value that has a fractional component when converted to its + string representation. For example, trying to set a byte size value + setting to a value of 1536 bytes is problematic because internally this + is converted to the string "1.5k". When we go to get this setting, we + try to parse "1.5k" back to a byte size value, which does not support + fractional values. The problem is that internally we are relying on a + method which loses the unit when doing the string conversion. Instead, + we are going to use a method that does not lose the unit and therefore + we can roundtrip from the byte size value to the string and back to the + byte size value. + +commit ad4e4fd9b88c7831752ff7a5038238c4d82d8072 +Author: Luca Cavanna +Date: Mon Jan 7 15:59:26 2019 +0100 + + Disable bwc tests until #36997 & #37142 backports are complete (#37187) + +commit 82b1f10eb143bbe36df3aab634758c7b31534fb9 +Author: Armin Braun +Date: Mon Jan 7 15:24:23 2019 +0100 + + SNAPSHOT+TESTS: Rem. Mock Atomic Writes Randomness (#37011) + + * Randomly doing non-atomic writes causes rare 0 byte reads from `index-N` files in tests + * Removing this randomness fixes these random failures and is valid because it does not reproduce a real-world failure-mode: + * Cloud-based Blob stores (S3, GCS, and Azure) do not have inconsistent partial reads of a blob, either you read a complete blob or nothing on them + * For file system based blob stores the atomic move we do (to atomically write a file) by setting `java.nio.file.StandardCopyOption#ATOMIC_MOVE` would throw if the file system does not provide for atomic moves + * Closes #37005 + +commit edb48321bae4efc7d4549eb48d38959f4bc739bf +Author: Josh Soref +Date: Mon Jan 7 08:44:12 2019 -0500 + + [DOCS] Various spelling corrections (#37046) + +commit ac2e09b25abc2319e86ea74492edb54458690019 +Author: Simon Willnauer +Date: Mon Jan 7 14:20:17 2019 +0100 + + Fix suite scope random initializaation (#37163) + + The initialization of a suite scope cluster had some sideffects on + subsequent runs which causes issues when tests must be reproduced. + This moves the suite scope initialization to a privte random context. + + Closes #36202 + +commit 21996f76908f6247cbbe1eb75302de39723bfc44 +Author: Alpar Torok +Date: Mon Jan 7 14:06:06 2019 +0200 + + Expand on #37058 + + Use the wrapper + +commit f5af79b9cdd9df393e4073f5b67cf9da82f4f0ee +Merge: 19593884eef 617e2941338 +Author: Tanguy Leroux +Date: Mon Jan 7 12:43:03 2019 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 617e29413387f7ddd83c841705d7d9d50838a27b +Author: Armin Braun +Date: Mon Jan 7 12:11:19 2019 +0100 + + SNAPSHOT: Make Atomic Blob Writes Mandatory (#37168) + + * With #37066 introducing atomic writes to HDFS repository we can enforce atomic write capabilities on this interface + * The overrides on the other three cloud implementations are ok because: + * https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html states that "Amazon S3 never adds partial objects; if you receive a success response, Amazon S3 added the entire object to the bucket." + * https://cloud.google.com/storage/docs/consistency states that GCS has strong read-after-write consistency + * https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#remarks Azure has the concept of committing blobs, so there's no partial content here either + * Relates #37011 + +commit 7cc749dcedd7241d717abd6d5855b15a82156cbf +Author: David Kyle +Date: Mon Jan 7 09:47:40 2019 +0000 + + Mute SearchStatsIT.testOpenContexts + +commit 60742b12e9e2bcf168e5b45438d0d5d2c28f12c6 +Author: Christoph Büscher +Date: Mon Jan 7 10:42:07 2019 +0100 + + Remove deprecated CardinalityAggregationBuilder#rehash method (#37154) + + It has been deprecated since at least 6.0, is a no-op and unused in the rest of + our code. + +commit d18c3d651d55c4b072473097afa0ac21a8fb1170 +Author: Christoph Büscher +Date: Mon Jan 7 10:41:40 2019 +0100 + + Introduce an `include_type_name` constant (#37155) + + I started referring to this parameter name from various places in #37149 so I + think it's a good idea to simplify things by referring to a common constant. + +commit 12a105e5ef1467f792ea8fda346c66e4da1c6b8d +Author: Christoph Büscher +Date: Mon Jan 7 10:41:04 2019 +0100 + + Remove deprecated PutIndexTemplateRequestBuilder#setTemplate (#37151) + + The method has been removed since 6.0, there is a direct replacement and it is + only used in tests still. + +commit 2f4dafa69f38f11742f5010bf0957304961c9191 +Author: Luca Cavanna +Date: Mon Jan 7 10:28:31 2019 +0100 + + Add support for providing absolute start time to SearchRequest (#37142) + + We have recently added support for providing a local cluster alias to a + SearchRequest through a package protected constructor. When executing + cross-cluster search requests with local reduction on each cluster, the + CCS coordinating node will have to provide such cluster alias to each + remote cluster, as well as the absolute start time of the search action + in milliseconds from the time epoch, to be used when evaluating date + math expressions both while executing queries / scripts as well as when + resolving index names. + + This commit adds support for providing the start time together with the + cluster alias. It is a final member in the search request, which will + only be set when using cross-cluster search with local reduction (also + known as alternate execution mode). When not provided, the coordinating + node will determine the current time and pass it through (by calling + `System.currentTimeMillis`). + + Relates to #32125 + +commit 19593884eef94392a48d5ba05c0255edb4ea0715 +Author: Tanguy Leroux +Date: Mon Jan 7 09:43:50 2019 +0100 + + [Close Index API] Propagate tasks ids between Freeze, Close and Verify Shard actions (#36630) + + This pull request changes the Freeze Index and Close Index actions so + that these actions always requires a Task. The task's id is then propagated + from the Freeze action to the Close action, and then to the Verify shard action. + This way it is possible to track which Freeze task initiates the closing of an index, + and which consecutive verifiy shard are executed for the index closing. + +commit 63474611464d8553a8695d90c3e444026060ddfb +Author: Tanguy Leroux +Date: Mon Jan 7 09:42:27 2019 +0100 + + Rename ClusterBlocks.hasGlobalBlock methods (#36941) + + As suggested in #36775, this pull request renames the following methods: + + ClusterBlocks.hasGlobalBlock(int) + ClusterBlocks.hasGlobalBlock(RestStatus) + ClusterBlocks.hasGlobalBlock(ClusterBlockLevel) + + to something that better reflects the property of the ClusterBlock that is searched for: + + ClusterBlocks.hasGlobalBlockWithId(int) + ClusterBlocks.hasGlobalBlockWithStatus(RestStatus) + ClusterBlocks.hasGlobalBlockWithLevel(ClusterBlockLevel) + +commit 31c33fdb9b70de5935522f4fd455edb319f33d59 +Author: Armin Braun +Date: Mon Jan 7 09:21:25 2019 +0100 + + MINOR: Remove some Deadcode in Gradle (#37160) + +commit 39a072389c92b0a160adbc25d010e96f2795ff0c +Author: Andrei Stefan +Date: Mon Jan 7 10:08:51 2019 +0200 + + SQL: add sub-selects to the Limitations page (#37012) + +commit e778abaac58b598c2c9f3d844ed70761b773461b +Author: Marios Trivyzas +Date: Mon Jan 7 09:21:23 2019 +0200 + + SQL: Improve error message when unable to translate to ES query DSL (#37129) + + Improve error message returned to the client when an SQL statement + cannot be translated to a ES query DSL. Cases: + + 1. WHERE clause evaluates to FALSE => No results returned + 1. Missing FROM clause => Local execution, e.g.: SELECT SIN(PI()) + 3. Special SQL command => Only valid of SQL iface, e.g.: SHOW TABLES + + Fixes: #37040 + +commit bf5bc88f50d55fd5474de1694548a9c82aba4e86 +Author: Jason Tedor +Date: Sun Jan 6 22:34:52 2019 -0800 + + Fix handling of fractional time value settings (#37171) + + This commit addresses an issue when setting a time value setting using a + value that has a fractional component when converted to its string + representation. For example, trying to set a time value setting to a + value of 1500ms is problematic because internally this is converted to + the string "1.5s". When we go to get this setting, we try to parse + "1.5s" back to a time value, which does not support fractional + values. The problem is that internally we are relying on a method which + loses the unit when doing the string conversion. Instead, we are going + to use a method that does not lose the unit and therefore we can + roundtrip from the time value to the string and back to the time value. + +commit da3d8fb5b74e99b553e4ab4e04ad8b3fd294e4b3 +Author: Marios Trivyzas +Date: Sun Jan 6 18:29:34 2019 +0200 + + SQL: Fix issue with wrong NULL optimization (#37124) + + Logical operators OR and AND as well as conditional functions + (COALESCE, LEAST, GREATEST, etc.) cannot be folded to NULL if one + of their children is NULL as is the case for most of the functions. + Therefore, their nullable() implementation cannot return true. On + the other hand they cannot return false as if they're wrapped within + an IS NULL or IS NOT NULL expression, the expression will be folded + to false and true respectively leading to wrong results. + + Change the signature of nullable() method and add a third value UKNOWN + to handle these cases. + + Fixes: #35872 + +commit b34e7d4f192c089c202465ecbe7aed730aaa1a02 +Author: Armin Braun +Date: Sat Jan 5 17:52:24 2019 +0100 + + SNAPSHOT+TESTS: Relax Assertion in DisruptionIT (#37144) + + * The retries on the failing master can lead to concurrently trying to create and delete a snapshot, catch this for now to fix this test + * closes #36779 + +commit 0cc877026f8cdc43dc1a0d3801ea18a40e8a9603 +Author: Simon Willnauer +Date: Sat Jan 5 10:15:33 2019 +0100 + + Subclass NIOFSDirectory instead of using FileSwitchDirectory (#37140) + + We don't want two FSDirectories manage pending deletes separately + and optimize file listing. This confuses IndexWriter and causes exceptions + when files are deleted twice but are pending for deletion. This change + move to using a NIOFS subclass that only delegates to MMAP for opening files + all metadata and pending deletes are managed on top. + + Closes #37111 + Relates to #36668 + +commit 0bac64fbd3ae38bab467056073cd88977332e38a +Author: Julie Tibshirani +Date: Fri Jan 4 13:05:52 2019 -0800 + + Deprecate the _type field in aggregations. (#37131) + +commit e40193ae66aab511f47d102810daa4442e42e35b +Author: Michael Basnight +Date: Fri Jan 4 13:37:59 2019 -0600 + + HLRC: Fix Reindex from remote query logic (#36908) + + The query object was incorrectly added to the remote object in the + xcontent. This fix moves the query back into the source, if it was + passed in as part of the RemoteInfo. It also adds a IPv6 test for + reindex from remote such that we can properly validate this. + +commit e38cf1d0dca1776d9659b4c0f3c010ba444642ca +Author: Jim Ferenczi +Date: Fri Jan 4 20:36:49 2019 +0100 + + Add the ability to set the number of hits to track accurately (#36357) + + In Lucene 8 searches can skip non-competitive hits if the total hit count is not requested. + It is also possible to track the number of hits up to a certain threshold. This is a trade off to speed up searches while still being able to know a lower bound of the total hit count. This change adds the ability to set this threshold directly in the track_total_hits search option. A boolean value (true, false) indicates whether the total hit count should be tracked in the response. When set as an integer this option allows to compute a lower bound of the total hits while preserving the ability to skip non-competitive hits when enough matches have been collected. + + Relates #33028 + +commit ac4aecc92d5201eb9d7cb452028775dc38f68007 +Author: Adam Thomson +Date: Fri Jan 4 18:20:32 2019 +0000 + + [Docs] Update ingest-node.asciidoc (#37116) + +commit b4f113d3ea052ed2d3e56e263154bdab3bac25f9 +Author: Simon Willnauer +Date: Fri Jan 4 13:51:06 2019 +0100 + + Don't block on peer recovery on the target side (#37076) + + Today we block using the generic thread-pool on the target side + until the source side has fully executed the recovery. We still + block on the source side executing the recovery in a blocking fashion + but there is no reason to block on the target side. This will + release generic threads early if there are many concurrent recoveries + happen. + + Relates to #36195 + +commit 41d7e3a2fe9de55de45e55426ffcb74b96775653 +Author: Simon Willnauer +Date: Fri Jan 4 13:49:40 2019 +0100 + + Expose `search.throttled` on `_cat/indices` (#37073) + + Today it's very difficult to see which indices are frozen or rather + throttled via the commonly used monitoring APIs. This change adds + a cell to the `_cat/indices` API to render if an index is `search.throttled` + + Relates to #34352 + +commit ff7df40b20f25ffb354cae540632062f4a6957e0 +Author: David Roberts +Date: Fri Jan 4 12:21:28 2019 +0000 + + [ML] Uplift model memory limit on job migration (#37126) + + When a 6.1-6.3 job is opened in a later version + we increase the model memory limit by 30% if it's + below 0.5GB. The migration of jobs from cluster + state to the config index changes the job version, + so we need to also do this uplift as part of that + config migration. + + Relates #36961 + +commit 21d52f0dab10575b59bfffc9b473cf1b7ea6e6a4 +Author: Luca Cavanna +Date: Fri Jan 4 12:19:31 2019 +0100 + + Ensure that local cluster alias is never treated as remote (#37121) + + With #36997 we added support for providing a local cluster alias with a + `SearchRequest`. We intended to make sure that when provided as part of + a search request, the cluster alias would never be used for connection + lookups. Yet due to a bug we would still end up looking up the + connection from the remote ones. + + This commit adds a test to make sure that whenever we set the cluster + alias to the `SearchRequest` (which can only be done at transport), such + alias is used as index prefix in the returned hits. No errors are thrown + despite no remote clusters are configured indicating that such alias is + never used for connection look-ups. + + Also, we add explicit support for the empty cluster alias when printing + out index names through `RemoteClusterAware#buildRemoteIndexName`. + In fact we don't want to print out `:index` when the cluster alias is + set to empty string, but rather `index`. Yet, the semantic of empty + string is different compared to `null` as it will still disable final + reduction. This will be used in CCS when searching against remote + clusters as well as the local one, the local one will have empty prefix + yet it will need to disable final reduction so that its results will be + properly merged with the ones coming from the remote clusters. + +commit 0fd27d4d6f44264b9b13ad6e9125eb7830ca11ac +Author: Dimitris Athanasiou +Date: Fri Jan 4 12:36:51 2019 +0200 + + [ML] Unused state remover should also account for jobs in index (#37119) + + The unused state remover was never adjusted to account for jobs stored + in the config index. The result was that when triggered it removed + state for all jobs stored in the config index. + This commit fixes the issue. + + Closes #37109 + +commit bfe6f091da53fe6c364be97fb8be1d7e85ef13b8 +Author: Luca Cavanna +Date: Fri Jan 4 10:38:20 2019 +0100 + + [DOCS] fix typo in 7.0.0-alpha2 release notes + +commit 3f7d6a989af09fb61a8904816ac644ddae1cdd46 +Author: David Turner +Date: Fri Jan 4 09:09:16 2019 +0000 + + [Zen2] Elect freshest master in upgrade (#37122) + + Today when electing a master in Zen2 we use the cluster state version to + determine whether a node has a fresh-enough cluster state to become master. + However the cluster state version is not a reliable measure of freshness in the + Zen1 world; furthermore in 6.x the cluster state version is not persisted. This + means that when upgrading from 6.x via a full cluster restart a cluster state + update may be lost if a stale master wins the initial election. + + This change fixes this by using the metadata version as a measure of freshness + when in term 0, since this is persisted in 6.x and does more reliably indicate + the freshness of nodes. + + It also makes changes parallel to elastic/elasticsearch-formal-models#40 to + support situations in which nodes accept cluster state versions in term 0: this + does not happen in a pure Zen2 cluster, but can happen in mixed clusters and + during upgrades. + +commit 586453fef10ec1a74853653dd7301227be3859a8 +Author: Dimitris Athanasiou +Date: Fri Jan 4 09:43:44 2019 +0200 + + [ML] Remove types from datafeed (#36538) + + Closes #34265 + +commit 00997b4f60f9bda54f925b732fef4ad9d0410f71 +Author: lcawley +Date: Tue Apr 17 08:48:08 2018 -0700 + + [DOCS] Fixes broken links + +commit 7839cec3015468ecbb525f390e3fa2381cb1817d +Author: Peter Dyson +Date: Fri Jan 4 17:34:24 2019 +1000 + + subsequent fix to edit in recent cherry-pick + +commit 7cc9754d940b12601dcba20a1a3a88639e94b10e +Author: Peter Dyson +Date: Fri Jan 4 17:26:42 2019 +1000 + + fix to edit in recent cherry-pick + +commit 3578e69669c7ece8ca7f31f3be018b359c7f8a49 +Author: Andrei Stefan +Date: Fri Jan 4 09:19:24 2019 +0200 + + SQL: Handle the bwc Joda ZonedDateTime scripting class in Painless (#37024) + + * Handle the bwc Joda ZonedDateTime scripting class in Painless + + * Integrated the types checking in the already existent method + +commit 0ff2707c9f40d24692f6b61155c6d6d806a8c5da +Author: Peter Dyson +Date: Tue Apr 17 10:22:20 2018 +1000 + + Add Profile API to search speed tuning howto (#29489) + + * Add Profile API to search speed tuning howto + + Seemed useful to mention the Profile API in the context of tuning for search speed. + +commit 3c12c372d0971f1e00b711e46ac51a8420491ba6 +Author: Ioannis Kakavas +Date: Fri Jan 4 08:26:41 2019 +0200 + + [DOCS] Adjust role mapping docs for SAML (#37083) + + Explicitly mention that file based role mappings cannot be used with + the SAML realm. + +commit ac1c6940d20b9ac993422b279d8f3a8bd73a3527 +Author: Julie Tibshirani +Date: Thu Jan 3 17:41:28 2019 -0800 + + Stop automatically nesting mappings in index creation requests. (#36924) + + Now that we unwrap mappings in DocumentMapperParser#extractMappings, it is not + necessary for the mapping definition to always be nested under the type. This + leniency around the mapping format was added in 2341825358e740b0bea4c16d164c5acdf12fc6b3. + +commit eaeccd840104e8ac96ca200702dccd87517a1b50 +Author: Tal Levy +Date: Thu Jan 3 15:00:40 2019 -0800 + + [ILM] Add Freeze Action (#36910) + + This commit adds a new ILM Action for + freezing indices in the cold phase. + + Closes #34630. + +commit 40a30c6f5f1dfc9ae4251453d5a1f4400daf7767 +Author: Costin Leau +Date: Fri Jan 4 00:57:50 2019 +0200 + + SQL: Preserve original source for each expression (#36912) + + Improve parsing to save the source for each token alongside the location + of each Node/Expression for accurate reproducibility of an expression + name and source + + Fix #36894 + +commit 7686ee763183e4dca0d4373f9b432e7ac87c1625 +Author: Armin Braun +Date: Thu Jan 3 22:35:44 2019 +0100 + + TESTS: Shutdown ThreadPool after TestNodes (#37123) + + * If the threadpool gets shut down before the testnodes we run into an error => fixed by moving to single `After` method + * Relates #36976 + +commit 2c3ce5b9ded43ed40470a8fa29267d2a34d94fda +Author: Benjamin Trent +Date: Thu Jan 3 14:59:42 2019 -0600 + + [ML] add min_version field to machine learning results index mapping (#37115) + + * ML: add min_version field to mapping + + * Adding min_version as reserved field name + +commit d6acd8e15f2f7803f8ef011b81c45d637944dec6 +Author: Igor Motov +Date: Thu Jan 3 15:40:48 2019 -0500 + + Docs: add clarification about geohash use in geohashgrid agg (#36901) + + Adds an example on translating geohashes returned by geohashgrid + agg as bucket keys into geo bounding box filters in elasticsearch as well + as 3rd party applications. + + Closes #36413 + +commit 54f53d2a518f7301aeac53dbc59b42ab5ad33334 +Author: Julie Tibshirani +Date: Thu Jan 3 11:53:08 2019 -0800 + + Make sure to accept empty unnested mappings in create index requests. (#37089) + +commit e613bcae43deae751e64b12bf857835ac7fe07b6 +Author: Nick Knize +Date: Thu Jan 3 12:48:36 2019 -0600 + + Remove XLatLonShape classes (#37094) + + This commit removes local XLatLonShape classes and replaces with current LatLonShape classes in latest lucene snapshot + +commit e4ec51879e68997b5428bba87457f84aaa52baa7 +Author: Daniel Mitterdorfer +Date: Thu Jan 3 19:27:18 2019 +0100 + + Retry JDK download when building Docker image + + With this commit we instruct curl to retry with a backoff when + downloading the JDK for the Elasticsearch Docker image. This avoids + build failures on transient network issues. Note that this option + requires curl 7.12.3 or better. + + Relates #37103 + Relates #37113 + +commit de962b2f39ed21323eb304eed91c2e29f3c3febd +Author: Nicholas Knize +Date: Thu Jan 3 11:52:31 2019 -0600 + + Revert "Adjust Lucene version for 6.7" + + This reverts commit b7f6ee72a68324b8fc6b523b1eafb991544d2278. + +commit 366b6b5e11c249e804ccfde50a5d2145a7965e40 +Author: Lisa Cawley +Date: Thu Jan 3 09:26:57 2019 -0800 + + [DOCS] Adds cluster setting to enable/disable config migration (#36948) + +commit fc49d88b92e20c4352b1eae935767d037a3677bb +Author: David Kyle +Date: Thu Jan 3 17:15:35 2019 +0000 + + [ML] Adjust migration full cluster restart tests based on old cluster version #37118 + +commit 13649aa70a9237bef857cd51255a9312ad53b085 +Author: David Roberts +Date: Thu Jan 3 16:23:18 2019 +0000 + + [TEST] Revert "Mute ForecastIT.testSingleSeries" (#37110) + + The problem that caused the test to be muted was fixed in + https://github.com/elastic/ml-cpp/pull/332 + + Closes #36258 + +commit 99f13b90d34c9e3be47fd1859241fee068fdfa1b +Author: Armin Braun +Date: Thu Jan 3 16:16:05 2019 +0100 + + SNAPSHOT: Speed up HDFS Repository Writes (#37069) + + * There is no point in hsyncing after every individual write since there is only value in completely written blobs for restores, this is ensures by the `SYNC` flag already and there is no need for separately invoking `hsync` after individual writes + +commit 10d9819f99c5cf27dc72fbaae3fd639baf161308 +Author: Armin Braun +Date: Thu Jan 3 15:51:47 2019 +0100 + + Implement Atomic Blob Writes for HDFS Repository (#37066) + + * Implement atomic writes the same way we do for the FsBlobContainer via rename which is atomic + * Relates #37011 + +commit 675ea4c59cf842d5c835aba25e92d7d104f61d45 +Author: Armin Braun +Date: Thu Jan 3 15:19:21 2019 +0100 + + TESTS: Remove Static Threadpool in TaskManagerTest (#36976) + + * The static threadpool leaks a lot of memory in these tests because it prevents things like the connect listeners from `org.elasticsearch.transport.TcpTransport#initiateConnection` + to be GCed between tests (since they keep being referenced by the threadpool) which in turn reference channels and their underlying buffers + * I could not find any slowdown in executing these tests from this change, if anything they are slightly faster now on my machine + * Relates #36906 (which may be caused by slowness from leaking memory and also becomes testable in a loop by this change) + +commit cfc310748deda29ccb55d782d93e1687e207dd23 +Author: Benjamin Trent +Date: Thu Jan 3 07:25:57 2019 -0600 + + addressing (#36891)(#36888)(#36889) (#37080) + +commit b04b3173db3dd69ba1e70a388f8f8d609e6e6932 +Author: Dimitris Athanasiou +Date: Thu Jan 3 15:15:31 2019 +0200 + + [ML][TEST] Clean up max_model_memory_limit cluster setting (#37101) + + Removes the `xpack.ml.max_model_memory_limit` cluster setting + at the teardown of the `ml_info.yml` tests to ensure the setting + does not trip other tests. + +commit 33137907cf7dd6cdc81bbc22a977f55ac0436b85 +Author: Marios Trivyzas +Date: Thu Jan 3 13:55:09 2019 +0200 + + SQL: Enhance message for PERCENTILE[_RANK] with field as 2nd arg (#36933) + + Enhance error message for the case that the 2nd argument of PERCENTILE + and PERCENTILE_RANK is not a foldable, as it doesn't make sense to have + a dynamic value coming from a field. + + Fixes: #36903 + +commit 046f86f2748e5536690540bf33618f2d64927cb5 +Author: Christoph Büscher +Date: Thu Jan 3 10:29:14 2019 +0100 + + Deprecate use of type in reindex request body (#36823) + + Types can be used both in the source and dest section of the body which will + be translated to search and index requests respectively. Adding a deprecation warning + for those cases and removing examples using more than one type in reindex since + support for this is going to be removed. + +commit e21054d176715c2e26584bec6630254b74d2c045 +Author: Christoph Büscher +Date: Thu Jan 3 10:28:47 2019 +0100 + + Remove two unused methods in Iterables (#37075) + + These helper methods are unused in the rest of the codebase. + +commit b7f6ee72a68324b8fc6b523b1eafb991544d2278 +Author: Nhat Nguyen +Date: Thu Jan 3 04:20:47 2019 -0500 + + Adjust Lucene version for 6.7 + + Relates #37088 + +commit 78ba1889cfedcf859f0662e7425ce1e0b474ea65 +Author: Jim Ferenczi +Date: Thu Jan 3 09:51:35 2019 +0100 + + Replace the TreeMap in the composite aggregation (#36675) + + The `composite` aggregation uses a TreeMap to keep track of the best buckets. + This ensures a log(n) time cost to insert new buckets but also to retrieve buckets + that are already present in the map. In order to speed up the retrieval of buckets + this change replaces the TreeMap with a priority queue and a HashMap. The insertion + cost is still log(n) but the retrieval of buckets through the HashMap is now done in constant + time. This optimization can bring significant improvement since each document needs + to check if its associated buckets are already present in the current best buckets. + +commit a40f0545e65e22ee045c5f71d4622ffa77e44887 +Author: David Turner +Date: Thu Jan 3 08:28:49 2019 +0000 + + Make ILM examples larger (#37039) + + There are a handful of examples in the ILM documentation that could result in + rolling over indices more quickly than we might normally recommend, + contributing to over-sharding in cases where the examples are copied without + modification. This change makes some numbers bigger to try and avoid this. + +commit 75f3443c62e7319f1c69c208980b12e7b4611255 +Author: Daniel Mitterdorfer +Date: Thu Jan 3 07:10:34 2019 +0100 + + Rename setting to enable mmap + + With this commit we rename `node.store.allow_mmapfs` to + `node.store.allow_mmap`. Previously this setting has controlled whether + `mmapfs` could be used as a store type. With the introduction of + `hybridfs` which also relies on memory-mapping, + `node.store.allow_mmapfs` also applies to `hybridfs` and thus we rename + it in order to convey that it is actually used to allow memory-mapping + but not a specific store type. + + Relates #36668 + Relates #37070 + +commit b2aa655f469f10ce75595ec21529c7746c5f922d +Author: Nick Knize +Date: Wed Jan 2 20:18:19 2019 -0600 + + Upgrade master to lucene-8.0.0-snapshot-a1c6e642aa (#37091) + + Updates the master branch to the latest snapshot of Lucene 8.0. + +commit f0cd19f39b083759a883a6c2fa6d1616abaebd84 +Author: Ioannis Kakavas +Date: Wed Jan 2 23:50:55 2019 +0200 + + [TEST] Lower deleteInterval in TokenAuthIntegTests + + We run subsequent token invalidation requests and we still want to + trigger the deletion of expired tokens so we need to lower the + deleteInterval parameter significantly. Especially now that the + bwc expiration logic is removed and the invalidation process is + much shorter + + Resolves #37063 + +commit d4e766024846ad9578f5d0621d9dc92ec6a72085 +Author: David Findley +Date: Wed Jan 2 13:53:21 2019 -0800 + + Fix weighted_avg parser not found for RestHighLevelClient (#37027) + + Add integration test for weighted avg sub aggregation + Add weighted avg parser to DefaultNamedXContents + + Fixes #36861 + +commit 62ece69b929118b386e75a8b49c0cbb0a702aacc +Author: Ke Li +Date: Thu Jan 3 05:01:12 2019 +0800 + + Remove system property es.enforce_max_shards_per_node (#36968) + + The system property es.enforce_max_shards_per_node was not needed any more, + because we always enforce cluster-wide shard limit now. + +commit 7a0047744dc6e047ead2391cf49268dae0fe90c3 +Author: Alan Woodward +Date: Wed Jan 2 20:12:24 2019 +0000 + + `query_string` should use indexed prefixes (#36895) + + The QueryStringQueryBuilder does not currently delegate to the field mapper's prefixQuery + method, so does not use indexed prefixes. This commit corrects this. + + It also fixes a bug where a query a* would not match the word a if indexed prefixes were used with + a minchar setting of 2. + +commit 265fdce312fbe8301d7f563eeaf9c0b8da6bfd61 +Author: Julie Tibshirani +Date: Wed Jan 2 10:46:20 2019 -0800 + + Fix FullClusterRestartIT#testRollupIDSchemeAfterRestart by adding a missing expected warning. + +commit 42bb2bae216af113c72059095ad3850f6f24aa7b +Author: David Kyle +Date: Wed Jan 2 16:52:20 2019 +0000 + + [ML] Order GET job stats response by job id (#36841) + +commit 42ea6449033d82902a86285ae6db84412c8c7f35 +Author: Luca Cavanna +Date: Wed Jan 2 17:45:49 2019 +0100 + + Remove single shard optimization when suggesting shard_size (#37041) + + When executing terms aggregations we set the shard_size, meaning the + number of buckets to collect on each shard, to a value that's higher than + the number of requested buckets, to guarantee some basic level of + precision. We have an optimization in place so that we leave shard_size + set to size whenever we are searching against a single shard, in which + case maximum precision is guaranteed by definition. + + Such optimization requires us access to the total number of shards that + the search is executing against. In the context of cross-cluster search, + once we will introduce multiple reduction steps (one per cluster) each + cluster will only know the number of local shards, which is problematic + as we should only optimize if we are searching against a single shard in a + single cluster. It could be that we are searching against one shard per cluster + in which case the current code would optimize number of terms causing + a loss of precision. + + While discussing how to address the CCS scenario, we decided that we do + not want to introduce further complexity caused by this single shard + optimization, as it benefits only a minority of cases, especially when + the benefits are not so great. + + This commit removes the single shard optimization, meaning that we will + always have heuristic enabled on how many number of buckets to collect + on the shards, even when searching against a single shard. + + This will cause more buckets to be collected when searching against a single + shard compared to before. If that becomes a problem for some users, they + can work around that by setting the shard_size equal to the size. + + Relates to #32125 + +commit e0a677b03310566431f7ee74b88917f8fed776b0 +Author: Josh Soref +Date: Wed Jan 2 11:40:11 2019 -0500 + + Clarify what to run for gradle idea (#37058) + +commit 02aa9f51eead29d094304e1d7cb362a9ad2d95d9 +Author: Josh Soref +Date: Wed Jan 2 11:38:43 2019 -0500 + + Spelling: correct wrong spellings of similar (#37054) + +commit f6e4b9a014be990c79d8a0fa7f25fce24ee02cf8 +Author: Josh Soref +Date: Wed Jan 2 11:37:03 2019 -0500 + + Spelling: correct wrong spellings of likelihood (#37052) + +commit 5d5d5c26bc3b36134249d6ae97b2e613ee5aa734 +Author: Josh Soref +Date: Wed Jan 2 11:35:53 2019 -0500 + + Spelling: replace interruptable with interruptible (#37049) + +commit c04d5c5468c9979eddf03a52412c03dfa2e29d58 +Author: Josh Soref +Date: Wed Jan 2 11:34:37 2019 -0500 + + Spelling: correct wrong spellings of "input" (#37048) + +commit 1df66d21fe7d818b8d98fe1adaf256dbfaf2e72c +Author: Josh Soref +Date: Wed Jan 2 11:33:02 2019 -0500 + + Spelling: replace uknown with unknown (#37056) + +commit 15c4d5b184d18b7e332c60e462d8b55b53ee2621 +Author: Przemyslaw Gomulka +Date: Wed Jan 2 17:22:15 2019 +0100 + + Fix line length in org.elasticsearch.get (#37071) + + Remove the line suppression for this package and fix offedning lines + relates #34844 + +commit 450d3014f654f7650a6b66e4535a6a518547b110 +Author: Luca Cavanna +Date: Wed Jan 2 16:46:41 2019 +0100 + + [TEST] Fix testLimitConcurrentShardRequests failure + + With #36221 we introduced shards counting to address a rare failure. + This caused a worse problem in this test when replicas were allocated + and shards failures were randomly returned. The latch has to take into + account additional attempts caused by the shard failures, which means + that in order for run to be called, performPhaseOnShard will be called + (numShards + numFailures) times. + + To address this, we need to decide upfront which shard is going to fail, + making sure that at least one shards is successful otherwise the whole + request fails. + + Closes #37074 + +commit b10a33054e8cf673ff01df072302de31dc019136 +Author: Luca Cavanna +Date: Wed Jan 2 16:35:07 2019 +0100 + + Mute failing testLimitConcurrentShardRequests + + relates to #37074 + +commit 8e6f68f6b01c67b9ccfa1cb8066e294813c5f933 +Author: Christoph Büscher +Date: Wed Jan 2 15:58:20 2019 +0100 + + Fix Eclipse specific type casting issue (#37072) + + The eclipse compiler needs some help in infering the functions return values. + +commit 51cb63f934f061ffa50ce767e4d04a5ca5aff842 +Author: Josh Soref +Date: Wed Jan 2 08:15:21 2019 -0500 + + Spelling: replace substract with subtract (#37055) + +commit 565a95c382e59760437b0895ae57307a5c869378 +Author: Josh Soref +Date: Wed Jan 2 08:13:54 2019 -0500 + + Spelling: replace respositories with repositories (#37053) + +commit d3e98278c375b3158d2521f99e3205a73ac27a03 +Author: Josh Soref +Date: Wed Jan 2 08:10:30 2019 -0500 + + Spelling: replace cachable with cacheable (#37047) + +commit 8a02bacf769645efbe948932cf41b98983105365 +Author: Abdullah DURSUN +Date: Wed Jan 2 12:32:42 2019 +0300 + + Fix typo in multi-search.asciidoc (#37060) + +commit f0052b1a7a49c17983e6b4a74ba1d34ab05c6027 +Author: Daniel Mitterdorfer +Date: Wed Jan 2 10:10:32 2019 +0100 + + Add hybridfs store type + + With this commit we introduce a new store type `hybridfs` that is a + hybrid between `mmapfs` and `niofs`. This store type chooses different + strategies to read Lucene files based on the read access pattern (random + or linear) in order to optimize performance. + + This store type has been available in earlier versions of Elasticsearch + as `default_fs`. We have chosen a different name now in order to convey + the intent of the store type instead of tying it to the fact whether it + is the default choice. + + Relates #36668 + +commit 9e706966282eaadafc77727059dd4194cbed492a +Author: Luca Cavanna +Date: Wed Jan 2 09:47:13 2019 +0100 + + [TEST] Address rejected execution in SearchAsyncActionTests (#37028) + + SearchAsyncActionTests may fail with RejectedExecutionException as InitialSearchPhase may try to execute a runnable after the test has successfully completed, and the corresponding executor was already shut down. The latch was located in getNextPhase that is almost correct, but does not cover for the last finishAndRunNext round that gets executed after onShardResult is invoked. + + This commit moves the latch to count the number of shards, and allowing the test to count down later, after finishAndRunNext has been potentially forked. This way nothing else will be executed once the executor is shut down at the end of the tests. + + Closes #36221 + Closes #33699 + +commit 35c09adbe1d003816e83b3095ea1818fbce22985 +Author: Alexis Wilke +Date: Tue Jan 1 07:37:15 2019 -0800 + + Replaced the word 'shards' with 'replicas' in an error message. (#36234) (#36275) + + Closes #36234 + +commit 794303af73ef721d5c0b54946b7b2145011d3df4 +Author: Peter Lyons +Date: Mon Dec 31 11:58:49 2018 -0500 + + [Docs] Fix URL path to aliases (#36852) + + This API URL contains /aliases/aliases and that seems to be a duplicate. + +commit 9c283aed0ad672ab7c427efd45e74ce2d5cfcd75 +Author: Andrei Stefan +Date: Mon Dec 31 14:07:39 2018 +0200 + + SQL: Move the SQL ODBC client applications pages from the ODBC section into the general ES SQL Client Applications page (#37004) + +commit 9a6c61015c1ac91e5ba7e826f3d63d2ecd9263bd +Author: Andrei Stefan +Date: Mon Dec 31 13:41:54 2018 +0200 + + Use the same format when comparing the error messages containing date fields (#37006) + + Test fix: make sure the format in which we check the date is the same as the one we used to format the initial JDBC date field. + +commit fd7cde88db5a7e2bf44543b215128261fb9da2eb +Author: Luca Cavanna +Date: Mon Dec 31 11:51:06 2018 +0100 + + Mute failing RolloverIT#testRolloverWithDateMath + + Relates to #37037 + +commit 85be9d6a896301a10bf412fcd756983c46168bf7 +Author: Armin Braun +Date: Mon Dec 31 11:17:21 2018 +0100 + + SNAPSHOT: Deterministic ClusterState Tests (#36644) + + * Use `DeterministicTaskQueue` infrastructure to test `SnapshotsService` + +commit adb957b5aab0ff7b756797c82e5acef451fc3451 +Author: Luca Cavanna +Date: Mon Dec 31 10:36:46 2018 +0100 + + Mute failing DateMathExpressionResolverTests tests + + Relates to #37037 + +commit 632c7fbed20165b526853739933eb1dcd0ccf31a +Author: Hendrik Muhs +Date: Mon Dec 31 08:30:08 2018 +0100 + + [ML] fix x-pack usage regression caused by index migration (#36936) + + Changes the feature usage retrieval to use the job manager rather than + directly talking to the cluster state, because jobs can now be either in + cluster state or stored in an index + + This is a follow-up of #36702 / #36698 + +commit d3f1fe46d361e6008098c56d756ce4530800db06 +Author: Luca Cavanna +Date: Fri Dec 28 17:03:40 2018 +0100 + + Increase await timeouts in RemoteClusterServiceTests + + Closes #33852 + +commit 4ac8fc6906a5441a1c7f0c0ffb0e5bf1f9769276 +Author: Armin Braun +Date: Fri Dec 28 16:42:51 2018 +0100 + + Force Refresh Listeners when Acquiring all Operation Permits (#36835) + + * Fixes the issue reproduced in the added tests: + * When having open index requests on a shard that are waiting for a refresh, relocating that shard + becomes blocked until that refresh happens (which could be never as in the test scenario). + +commit c1beb95aa1400e7caecfa4ae0997d33276dd4ca7 +Author: Luca Cavanna +Date: Fri Dec 28 16:39:36 2018 +0100 + + Mute LocalIndexFollowingIT#testRemoveRemoteConnection + + Relates to #37014 + +commit cb6bac3f8825163c585cbe2637d7a820d565eb2c +Author: Luca Cavanna +Date: Fri Dec 28 14:58:20 2018 +0100 + + Skip final reduction if SearchRequest holds a cluster alias (#37000) + + With #36997 we added the ability to provide a cluster alias with a + SearchRequest. + + The next step is to disable the final reduction whenever a cluster alias + is provided with the SearchRequest. A cluster alias will be provided + when executing a cross-cluster search request with alternate execution + mode, where each cluster does its own reduction locally. In order for + the CCS node to be able to later perform an additional reduction of the + results, we need to make sure that all the needed info stays available. + This means that terms aggregations can be reduced but not pruned, and + pipeline aggs should not be executed. The final reduction will happen + later in the CCS coordinating node. + + Relates to #36997 & #32125 + +commit 34d22f378d6546cce122fcc7767ffc0256260222 +Author: Armin Braun +Date: Fri Dec 28 14:20:45 2018 +0100 + + TESTS: Mute testSnapshotCanceledOnRemovedShard + + * relates #37005 + +commit 51fe20e0c30a422c8527bfba7c0e8ace10ba1ac4 +Author: Luca Cavanna +Date: Fri Dec 28 12:43:25 2018 +0100 + + Add support for local cluster alias to SearchRequest (#36997) + + With the upcoming cross-cluster search alternate execution mode, the CCS + node will be able to split a CCS request into multiple search requests, + one per remote cluster involved. In order to do that, the CCS node has + to be able to signal to each remote cluster that such sub-requests are + part of a CCS request. Each cluster does not know about the other + clusters involved, and does not know either what alias it is given in + the CCS node, hence the CCS coordinating node needs to be able to provide + the alias as part of the search request so that it is used as index prefix + in the returned search hits. + + The cluster alias is a notion that's already supported in the search shards + iterator and search shard target, but it is currently used in CCS as both + index prefix and connection lookup key when fanning out to all the shards. + With CCS alternate execution mode the provided cluster alias needs to be + used only as index prefix, as shards are local to each cluster hence no + cluster alias should be used for connection lookups. + + The local cluster alias can be set to the SearchRequest at the transport layer + only, and its constructor/getter methods are package private. + + Relates to #32125 + +commit 0cae979dfe14274e3d9a277a7ad20872f3c29d26 +Author: Ioannis Kakavas +Date: Fri Dec 28 13:09:42 2018 +0200 + + Remove bwc logic for token invalidation (#36893) + + - Removes bwc invalidation logic from the TokenService + - Removes bwc serialization for InvalidateTokenResponse objects as + old nodes in supported mixed clusters during upgrade will be 6.7 and + thus will know of the new format + - Removes the created field from the TokensInvalidationResult and the + InvalidateTokenResponse as it is no longer useful in > 7.0 + +commit 44bd7db59ee568a5e8d3b44219e9cf19ead356de +Author: Ioannis Kakavas +Date: Fri Dec 28 11:19:58 2018 +0200 + + [TEST] Pass a clientSSLContext in LdapTestCase + + If we don't explicitly sett the client SSLSocketFactory when + creating an InMemoryDirectoryServer and setting its SSL config, it + will result in using a TrustAllTrustManager(that extends + X509TrustManager) which is not allowed in a FIPS 140 JVM. + Instead, we get the SSLSocketFactory from the existing SSLContext + and pass that to be used. + + Resolves #37013 + +commit 935c2e98b0cec6fa560cd7cb65939cbe5ca7c895 +Author: Yannick Welsch +Date: Fri Dec 28 08:37:04 2018 +0100 + + Zen2: Turn to follower on follower check when no state accepted yet from new leader (#37003) + + Improves on #36449 which did not cover the situation where a node had bumped its term during + the election, and not when receiving the first follower check. This was uncovered while refactoring + NodeJoinTests so that they don't need to access to an internal field of Coordinator anymore (which + can now be made private). + +commit e16fd4ebd6ca95ef7f1eafa02eb576cb295648ba +Author: Tim Vernum +Date: Fri Dec 28 15:26:46 2018 +1100 + + Change missing authn message to not mention tokens (#36750) + + The phrase "missing authentication token" is historic and is based + around the use of "AuthenticationToken" objects inside the Realm code. + + However, now that we have a TokenService and token API, this message + would sometimes lead people in the wrong direction and they would try + and generate a "token" for authentication purposes when they would + typically just need a username:password Basic Auth header. + + This change replaces the word "token" with "credentials". + +commit 51c18c0e760c754f0f674f16936080b1ff609a0b +Author: Tim Vernum +Date: Fri Dec 28 11:31:34 2018 +1100 + + Reload SSL context on file change for LDAP (#36937) + + In #30509 we changed the way SSL configuration is reloaded when the + content of a file changes. As a consequence of that implementation + change the LDAP realm ceased to pick up changes to CA files (or other + certificate material) if they changed. + + This commit repairs the reloading behaviour for LDAP realms, and adds + a test for this functionality. + + Resolves: #36923 + +commit a02cfdf6e4f8fd9038a8c7a7d53d7a9918770e8a +Author: Andrey Ershov +Date: Thu Dec 27 22:21:37 2018 +0100 + + Switch InternalTestClusterTests to zen2 (#36977) + + Today InternalTestClusterTests is still using zen1. + This commit fixes it. + Two types of changes were required: + + 1. Explicitly pass file discovery host provider setting. It's done in + ESIntegTestCase as a part of the Zen2 feature and should be done here + as well. + 2. For the test, that uses autoManageMinMasterNodes = false perform + cluster bootstrap. + +commit 76da29ba845cc582db8c3bddb8b1b27146e1c48a +Author: Andrey Ershov +Date: Thu Dec 27 22:19:20 2018 +0100 + + Switch testClusterJoinDespiteOfPublishingIssues to Zen2 (#36979) + +commit 6259ccb1cf4e98ff6eba6b87bc77df5dad7641d0 +Author: Andrey Ershov +Date: Thu Dec 27 22:18:47 2018 +0100 + + Move ensureAtomicMoveSupported to NodeEnvironment (#36975) + + Atomic move support is needed not only for GatewayMetaState to work correctly + +commit 6aae7c8516fed4104f91b35d10935d650ed0ceaa +Author: Armin Braun +Date: Thu Dec 27 12:18:17 2018 +0100 + + SNAPSHOT+TESTS: Speed up Snapshot IT (#36990) + + * This speeds up the test from an average 25s down to 7s runtime + * There is no need for artificially slowing down the snapshot to reproduce the issue of an out of sync routing table in practice. + Over hundreds of test runs the test's snapshot shard service still runs in the index not found exception every time reproducing this issue. + * Relates #36294 + +commit b0d2957e10e5a35fa2d71b95a05bec759e227ee5 +Author: Jim Ferenczi +Date: Thu Dec 27 12:00:22 2018 +0100 + + [TEST] Fix possible identical mutation in SearchHitTests#mutate + +commit 0b1cb5300ed88119b9098335db5e2cee6fa29f0f +Author: Andrei Stefan +Date: Thu Dec 27 11:55:20 2018 +0200 + + Move ODBC documentation to `elasticsearch` repository (#36955) + +commit 47f0a47f3d6e052510f1e97ef14144cbbc4d447d +Author: Damien Alexandre +Date: Thu Dec 27 09:18:28 2018 +0100 + + [Docs] Fix badly escaped keywords in migrate_7_0/search.asciidoc (#36964) + +commit 48b0908fc62a4047c8b254a3c3b52c973019561a +Author: Itamar Benjamin +Date: Thu Dec 27 09:51:24 2018 +0200 + + Make InternalComposite key comparable + + Keys are compared in BucketSortPipelineAggregation so making key type (ArrayMap) implement Comparable. Maps are compared using the entry set's iterator so ordered maps order is maintain. For each entry first comparing key then value. Assuming all keys are strings. When comparing entries' values if type is not identical and\or type not implementing Comparable, throwing exception. Not implementing equals() and hashCode() functions as parent's ones are sufficient. Tests included. + +commit 7580d9d925d864b25466d693637b3bd3092d44cf +Author: Nhat Nguyen +Date: Mon Dec 24 14:06:50 2018 -0500 + + Make SourceToParse immutable (#36971) + + Today the routing of a SourceToParse is assigned in a separate step + after the object is created. We can easily forget to set the routing. + With this commit, the routing must be provided in the constructor of + SourceToParse. + + Relates #36921 + +commit 531ae8b3ab080024e3c74d8051fdb7b0e8073de4 +Author: Andrey Ershov +Date: Mon Dec 24 16:26:08 2018 +0100 + + Remove ensureNoPre019State check (#36974) + + This check is no longer needed because it's already ES v7 + +commit bd41150338d6f4de6f61eb6e79cb59cfabd4e926 +Author: David Turner +Date: Mon Dec 24 09:29:08 2018 +0000 + + Move 'lost cluster state updates' issue to DONE (#36959) + + Relates #34714. + +commit 561b704129d161febc36635f05d368de0f98e089 +Author: Martijn van Groningen +Date: Mon Dec 24 10:16:38 2018 +0100 + + [CCR] AutoFollowCoordinator and follower index already created (#36540) + + The AutoFollowCoordinator should be resilient to the fact that the follower + index has already been created and in that case it should only update + the auto follow metadata with the fact that the follower index was created. + + Relates to #33007 + +commit 44fe265d82ef19c7d679751fccb1da24480ae145 +Author: Martijn van Groningen +Date: Mon Dec 24 07:53:51 2018 +0100 + + [CCR] Added auto_follow_exception.timestamp field to auto follow stats (#36947) + + Currently auto follow stats users are unable to see whether an auto follow + error was recent or old. The new timestamp field will help user distinguish + between old and new errors. + +commit 4fb62fcba638340e47cc4ca7df51825c7088a60d +Author: Martijn van Groningen +Date: Mon Dec 24 07:28:34 2018 +0100 + + Make CCR resilient against missing remote cluster connections (#36682) + + Both index following and auto following should be resilient against missing remote connections. + This happens in the case that they get accidentally removed by a user. When this happens + auto following and index following will retry to continue instead of failing with unrecoverable exceptions. + + Both the put follow and put auto follow APIs validate whether the + remote cluster connection. The logic added in this change only exists + in case during the lifetime of a follower index or auto follow pattern + the remote connection gets removed. This retry behavior similar how CCR + deals with authorization errors. + + Closes #36667 + Closes #36255 + +commit 1e6eff61480d496618644383ed96b1c7f126062c +Author: Jason Tedor +Date: Sun Dec 23 16:13:45 2018 -0500 + + Remove special handling for ingest plugins (#36967) + + We added some special handling for installing and removing the + ingest-geoip and ingest-user-agent plugins when we converted them to + modules. This special handling was done to minimize breaking users in a + minor release. However, do not want to maintain this behavior forever so + this commit removes that special handling in the master branch so that + starting with 7.0.0 this special handling will be gone. + +commit 40c7ae6181dcc00fafd559ff0736cb67ec09c69c +Author: Nhat Nguyen +Date: Sun Dec 23 15:14:21 2018 -0500 + + Rewrite SourceToParse with resolved docType (#36921) + + We introduce a typeless API in #35790 where we translate the default + docType "_doc" to the user-defined docType. However, we do not rewrite + the SourceToParse with the resolved docType. This leads to a situation + where we have two translog operations for the same document with + different types: + + - prvOp [Index{id='9LCpwGcBkJN7eZxaB54L', type='_doc', seqNo=1, + primaryTerm=1, version=1, autoGeneratedIdTimestamp=1545125562123}] + + - newOp [Index{id='9LCpwGcBkJN7eZxaB54L', type='not_doc', seqNo=1, + primaryTerm=1, version=1, autoGeneratedIdTimestamp=-1}] + + Closes #36769 + +commit 9137d92ca613b7fe40574789686575c4404089c1 +Author: Jason Tedor +Date: Sun Dec 23 08:59:18 2018 -0500 + + Refactor ingest node API docs (#36962) + + This commit is a simple refactoring of the ingest node API docs, + breaking each API into a single file for ease of maintaining. + +commit e2e82039d9f7c4e289156e1fa7d5f03d431f71b6 +Author: David Turner +Date: Sun Dec 23 09:50:40 2018 +0000 + + Add resiliency note on replica divergence (#36960) + +commit 7562768bd6aa449a3018b661aba35e9d321fe190 +Author: Jason Tedor +Date: Sat Dec 22 20:51:18 2018 -0500 + + Fix ingest cross-doc links + + This commit fixes some cross-doc links from the old ingest plugins page + to the new ingest processor pages that arose after converting + ingest-geoip and ingest-user-agent to modules. + +commit e14f27c0331ab67723e19bf335df0268ed342021 +Author: Jason Tedor +Date: Sat Dec 22 20:31:07 2018 -0500 + + Fix titles of GeoIP and User Agent processor docs + + This commit makes the titles of the new GeoIP and User Agent processor + docs look more like the titles of the docs for other processors. + +commit c1722c24f1997932850889e0a309a2bdc4a1c1bf +Author: Jason Tedor +Date: Sat Dec 22 20:27:52 2018 -0500 + + Fix another typo in comment in RemovePluginCommand + + This commit fixes another typo introduced when explaining that + ingest-geoip and ingest-user-agent receive special handling in + RemovePluginCommand. + +commit 02a1b2bd756a34dc9361a3d66882c92650c3d198 +Author: Jason Tedor +Date: Sat Dec 22 20:26:10 2018 -0500 + + Fix typos in comment in RemovePluginCommand + + This commit fixes some typos in a comment explaining the special + handling of ingest-geoip and ingest-user-agent in RemovePluginCommand. + +commit 1f574bd17a1db1f3e0887ca7eea31fbeecb76ea2 +Author: Jason Tedor +Date: Sat Dec 22 20:20:53 2018 -0500 + + Package ingest-user-agent as a module (#36956) + + This commit moves ingest-user-agent from being a plugin to being a + module that is packaged with Elasticsearch distributions. + +commit d238b2934cc1581d0309f5e9d88429b78fcfdab9 +Author: Nhat Nguyen +Date: Thu Dec 20 23:20:21 2018 -0500 + + TEST: Fix Engine#testRebuildLocalCheckpointTracker + + In this test, we verify that the LocalCheckpointTracker is initialized + with the operations of the safe commit. And the test fails because + Engine#Index does not implement the equals method (should not + implement as it consists of a mutable ParsedDocument). + + Closes #36470 + +commit 2ecb0a2821c80cc16785557a69232db528b35b11 +Author: Jason Tedor +Date: Sat Dec 22 14:01:23 2018 -0500 + + Fix link to using ingest-geoip + +commit 434021c3ecac579ab95b5b1f4dcb91bcbf923edd +Author: Jason Tedor +Date: Sat Dec 22 09:49:56 2018 -0500 + + Add placeholder ingest-geoip plugin page (#36958) + + This commit adds a placeholder ingest-geoip plugin page as there are + other components in the Elastic Stack that still refer to these + pages. These docs would be broken without this placeholder page forcing + teams responsible for those docs to scramble to fix the build over the + weekend before a holiday period. Instead, we add a placeholder page so + the docs build continues to function, and those teams can fix their docs + without the constraint of a broken build. We also cleanup a few minor + docs issues that were missed during the initial changes to convert + ingest-geoip to a module. + +commit 7e2bb9c1aab2e033027267b1e5174e9cf83b969a +Author: Jason Tedor +Date: Sat Dec 22 07:34:14 2018 -0500 + + Fix NPE in CachingUsernamePasswordRealm (#36953) + + This commit fixes an NPE in the CachingUsernamePasswordRealm when the + cache is disabled. + +commit e1717df0ac814caa26d3ecba54e8bc95a905628a +Author: Jason Tedor +Date: Sat Dec 22 07:21:49 2018 -0500 + + Package ingest-geoip as a module (#36898) + + This commit moves ingest-geoip from being a plugin to being a module + that is packaged with Elasticsearch distributions. + +commit 6781a29f9bb442b855d8c79be46578f706e407d1 +Author: Jason Tedor +Date: Fri Dec 21 18:05:50 2018 -0500 + + Move ingest-geoip default databases out of config (#36949) + + Today the default databases bundled with ingest-geoip are treated as + configuration files that we unbundle into the Elasticsearch + configuration directory. This can cause problems for users using our + Docker images if they bind mount over the configuration + directory. Additionally, it creates complexity when trying to convert + ingest-geoip to a module. This commit moves these databases out of the + configuration directory and instead loads from the plugin + directory. Further, custom databases can still be put into the + configuration directory. + +commit 25625d2407df990e362b5c28b071a2b769298f73 +Author: Tal Levy +Date: Fri Dec 21 13:55:21 2018 -0800 + + [ILM][DOCS] add extra scenario to policy update docs (#36871) + + This extra scenario describes the case where an updated + policy increases the current phase's `min_age`. Now, the + docs explicitly describe this scenario as to what is + expected -- old min_age is used. + + Closes #35356. + +commit c13a7bc04ac027a3662412a0aae3ef52ff01b350 +Author: Jack Conradson +Date: Fri Dec 21 13:42:07 2018 -0800 + + [Painless] Add String Casting Tests (#36945) + + This adds additional standard casting tests for String as the original type. This also cleans up the error messages in the String to char cast method. + +commit 09fa827adcef24acee4b2dad44da90cada9c0590 +Author: Andrei Stefan +Date: Fri Dec 21 23:25:54 2018 +0200 + + SQL: documentation improvements and updates (#36918) + + * Added Limitations page + * Made the aggregations page follow the common template for functions + * Modified all tables to have the first row's cells content centered + * Polishing in other various sections + +commit 33e9cf3892bff3a5af9953cd8a3a9f7c5f94a5a0 +Author: Lisa Cawley +Date: Fri Dec 21 11:24:48 2018 -0800 + + [DOCS] Merges list of discovery and cluster formation settings (#36909) + +commit c8a8391dfaeb9b98c9fab0749162700f9bc64048 +Author: Tim Brooks +Date: Fri Dec 21 10:14:00 2018 -0700 + + Only compress responses if request was compressed (#36867) + + This is a follow-up to some discussions around #36399. Currently we have + relatively confusing compression behavior where compression can be + configured for requests based on transport.compress or a specific + setting for a remote cluster. However, we can only compress responses + based on transport.compress as we do not know where a request is + coming from (currently). + + This commit modifies the behavior to NEVER compress responses based on + settings. Instead, a response will only be compressed if the request was + compressed. This commit also updates the documentation to more clearly + described transport level compression. + +commit 3f5dd792b33b03fbfeeb9c89940d97669b6100aa +Author: David Turner +Date: Fri Dec 21 16:09:35 2018 +0000 + + Remove duplicate paragraph (#36942) + +commit 7f3b9c8aa3aa6317ea4f2455735e543efa6efc7e +Author: David Turner +Date: Fri Dec 21 13:52:07 2018 +0000 + + Fix URI to cluster stats endpoint on specific nodes (#36784) + +commit 8e97bf9d063dead62c1ecf0e98d511ebd55381a8 +Author: Dimitrios Liappis +Date: Fri Dec 21 15:18:43 2018 +0200 + + Fix typo in unitTest task (#36930) + + Fix the typo introduced in #36311 causing CI failures with the + FipsJvm. + +commit bd2af2c4007af254371d1355076596f4d9a52340 +Merge: fb24469fe75 cfea2fd68c7 +Author: Tanguy Leroux +Date: Fri Dec 21 12:22:24 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit cfea2fd68c706ff68ffe3f2514f7b539e87b215e +Author: David Turner +Date: Fri Dec 21 11:10:48 2018 +0000 + + RecoveryMonitor#lastSeenAccessTime should be volatile (#36781) + + This local field is accessed on multiple threads and is nonvolatile so + theoretically could yield stale values. Not sure it does in practice. + +commit 4ded4717fe1fb32c214cbfc7f9ac9ee1114c6081 +Author: Martijn van Groningen +Date: Fri Dec 21 09:36:40 2018 +0100 + + [CCR] Add `ccr.auto_follow_coordinator.wait_for_timeout` setting (#36714) + + This setting controls the wait for timeout the autofollow coordinator + should use when setting cluster state requests to a remote cluster. + +commit 7cbf03c001acb2e0ec6220574f392a345af427b4 +Author: Michael Basnight +Date: Thu Dec 20 21:30:24 2018 -0600 + + Scripting: Remove deprecated params.ctx (#36848) + + When the script contexts were created in 6, the use of params.ctx was + deprecated. This commit cleans up that code and ensures that params.ctx + is null in both watcher script contexts. + + Relates: #34059 + +commit fba710469a3c28819f2b2678eff99826d48ae769 +Author: Julie Tibshirani +Date: Thu Dec 20 18:06:41 2018 -0800 + + Refactor the REST actions to clarify what endpoints are deprecated. (#36869) + +commit b6f59ffd713c00c347677809274a16b88d4f1177 +Author: Jason Tedor +Date: Thu Dec 20 20:47:56 2018 -0500 + + Add JDK 12 to CI rotation (#36915) + + Now that JDK 12 has entered rampdown phase one, it is time for us to add + JDK 12 to the CI rotation. This commit adds matrix parameters such that + we try to build with JDK 12, and we test running on JDK 12. + +commit 59da7c3cc4c7f99d81770079336f2cefaf79c479 +Author: Tim Vernum +Date: Fri Dec 21 12:02:02 2018 +1100 + + Improve error message for 6.x style realm settings (#36876) + + Realm settings were changed in #30241 in a non-BWC way. + If you try and start a 7.x node using a 6.x config style, then the + default error messages do not adequately describe the cause of + the problem, or the solution. + + This change detects the when realms are using the 6.x style and fails + with a specific error message. + + This detection is a best-effort, and will detect issues when the + realms have not been modified to use the 7.x style, but may not detect + situations where the configuration was partially changed. + + e.g. We can detect this: + + xpack.security.authc: + realms.pki1.type: pki + realms.pki1.order: 3 + realms.pki1.ssl.certificate_authorities: [ "ca.crt" ] + + But this (where the "order" has been updated, but the "ssl.*" has not) + will fall back to the standard "unknown setting" check + + xpack.security.authc: + realms.pki.pki1.order: 3 + realms.pki1.ssl.certificate_authorities: [ "ca.crt" ] + + Closes: #36026 + +commit d9b2ed61357bf1323759af16298473d145fcf91d +Author: Tim Brooks +Date: Thu Dec 20 17:43:12 2018 -0700 + + Send clear session as routable remote request (#36805) + + This commit adds a RemoteClusterAwareRequest interface that allows a + request to specify which remote node it should be routed to. The remote + cluster aware client will attempt to route the request directly to this + node. Otherwise it will send it as a proxy action to eventually end up + on the requested node. + + It implements the ccr clean_session action with this client. + +commit a9834cd5a64dd48d9c869551f9048d93032d5a07 +Author: Lisa Cawley +Date: Thu Dec 20 13:34:11 2018 -0800 + + [DOCS] Remove redundant ILM attributes (#36808) + +commit ac032a0b9d23edc6e932d03918a54002100007a4 +Author: Costin Leau +Date: Thu Dec 20 23:11:56 2018 +0200 + + SQL: Fix bug regarding histograms usage in scripting (#36866) + + Allow scripts to correctly reference grouping functions + Fix bug in translation of date/time functions mixed with histograms. + Enhance Verifier to prevent histograms being nested inside other + functions inside GROUP BY (as it implies double grouping) + Extend Histogram docs + +commit 4cd570593d721ff0a83f436fdc3fedaea115a6fc +Author: Tim Brooks +Date: Thu Dec 20 13:53:04 2018 -0700 + + Update index mappings when ccr restore complete (#36879) + + This is related to #35975. When the shard restore process is complete, + the index mappings need to be updated to ensure that the data in the + files restores is compatible with the follower mappings. This commit + implements a mapping update as the final step in a shard restore. + +commit 2f80d6f4d3c135556392aa596e85c905bcf27956 +Author: Nik Everett +Date: Thu Dec 20 14:49:22 2018 -0500 + + Docs: Bump version to alpha2 after release + +commit a64fea10e20117c8dae8a7b3eaefd9c5360911fa +Author: Michael Basnight +Date: Thu Dec 20 13:48:35 2018 -0600 + + Enable IPv6 URIs in reindex from remote (#36874) + + Reindex from remote was using a custom regex to dermine what URIs were + valid. This commit removes the custom regex and uses the java.net.URI + class instead, allowing IPv6 support without changing the existing + validation around a URI in reindex from remote. + +commit d00780d00c985e239f334b1a2e40a169efe6ab18 +Author: Michael Basnight +Date: Thu Dec 20 13:43:10 2018 -0600 + + Watcher: Remove unused local variable in doExecute (#36655) + +commit 32bed098bbcb7ed1695eed447fd99712f885944f +Author: lcawl +Date: Thu Dec 20 10:23:28 2018 -0800 + + [DOCS] Synchs titles of X-Pack APIs + +commit be573ab5e76f04e18ee5973563f57551997b7872 +Author: Jack Conradson +Date: Thu Dec 20 09:42:33 2018 -0800 + + [Painless] Casting Tests for Object and Number (#36804) + + This adds more casting tests with the original type as Object and then Number. Covers the entire set of possible numeric cases for these two types. + +commit d6d51348908cb92d5da0c03b7ae87fa8230cc969 +Author: David Turner +Date: Thu Dec 20 17:11:12 2018 +0000 + + Fix name of SettingsBasedHostsProviderIT (#36778) + + It is not a _host_ provider because it provides more than one host. + +commit f6c1e3f14f89f89b2307b1e6aeacd371cc5f85e9 +Author: Tal Levy +Date: Thu Dec 20 08:55:02 2018 -0800 + + [ILM][TEST] increase assertBusy timeout (#36864) + + the testFullPolicy and testMoveToRolloverStep tests + are very important tests, but they sometimes timeout + beyond the default 10sec wait for shrink to occur. + This commit increases one of the assertBusys to + 20 seconds + +commit 52d34e45e73c45e7b11e26397c5f23e2dc4108bc (es/security_authz_engine) +Author: David Turner +Date: Thu Dec 20 15:31:52 2018 +0000 + + [Zen2] Minor logging improvements (#36818) + + * Adds term number and greppable phrase 'coordinator becoming' to Coordinator + mode changes + * Adds term and version to messages from the ClusterApplier about master + changes + * Reduces some LeaderChecker messages to TRACE level + +commit b42074c1cca673c7246e3b54790808649bf91915 +Author: Martijn van Groningen +Date: Thu Dec 20 15:21:52 2018 +0100 + + [CCR] Report error if auto follower tries auto follow a leader index with soft deletes disabled (#36886) + + Currently if a leader index with soft deletes disabled is auto followed then this index is silently ignored. + This commit changes this behavior to mark these indices as auto followed and report an error, which is visible in auto follow stats. Marking the index as auto follow is important, because otherwise the auto follower will continuously try to auto follow and fail. + + Relates to #33007 + +commit ca92d74e7e7102ffe52a9b2d6db3c89696b6dc67 +Author: Andrey Ershov +Date: Thu Dec 20 15:20:33 2018 +0100 + + [Zen2] Change unsafe bootstrap nodes count to nodes list in tests (#36559) + + This commit modifies ESSingleNodeTestCase and ESIntegTestCase and + several concrete test classes to use node names when bootstrapping the + cluster. + + Today ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING + setting is used to bootstrap clusters in tests. Instead, we want to use + ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING and get rid of + the former setting eventually. + + There were two main problems when refactoring InternalTestCluster: + + 1. Nodes are created one-by-one in buildNode method. And node.name + is created in this method as well. It's not suitable for bootstrapping, + because we need to have the names of all master eligible nodes in + advance, before creating the node with bootstrapping configuration set. + We address this issue by separating buildNode into two methods: + getNodeSettings and buildNode. We first iterate over all nodes to + get nodes settings, then change the setting for the bootstrapping node + and then proceed with building the node. + 2. If autoManageMinMasterNodes = false, there is no way for the test to + set the list of bootstrapping nodes because node names are not known in + advance. This problem is solved by adding updateNodesSettings method + to NodeConfigurationSource and ESIntegTestCase (which could be + overridden by concrete integration test class). Once we have the list + of settings for all nodes, the integration test class is allowed to + update it. In our case, we update the + ClusterBootrstapService.INITIAL_MASTER_NODES_SETTING setting. + +commit 7b1dfeff2ecdaa1b01418444dc35e24edbc63093 +Author: Martijn van Groningen +Date: Thu Dec 20 15:08:04 2018 +0100 + + Renamed `WHITE_LISTED_SETTINGS` to `NON_REPLICATED_SETTINGS` + because the latter better describes the purpose of this field. + +commit 45a9756f1e2efd63abb2ee23d6407668b871cf78 +Author: Ioannis Kakavas +Date: Thu Dec 20 15:43:18 2018 +0200 + + Unmute SecurityDocumentationIT test + + Since https://github.com/elastic/elasticsearch/pull/36362 is merged + the invalidateTokensTest for the HLRC can be run again. + +commit 35911d8dd74cf913c91f0cc4d2d4e63d99b8d29f +Author: Jason Tedor +Date: Thu Dec 20 08:04:54 2018 -0500 + + Split the ingest processor docs into multiple files (#36887) + + This commit breaks the single ingest docs file into multiple files, + factoring out the processor docs into a documentation file per + processor. This will help make this content easier to maintain. + +commit 1a23417aeb0440ec99dbb11c3d4a3a59720ce207 +Author: David Turner +Date: Thu Dec 20 13:02:44 2018 +0000 + + [Zen2] Update documentation for Zen2 (#34714) + + This commit overhauls the documentation of discovery and cluster coordination, + removing mention of the Zen Discovery module and replacing it with docs for the + new cluster coordination mechanism introduced in 7.0. + + Relates #32006 + +commit 08bcd8375783efe2144056410c678128d7cc91aa +Author: Dimitris Athanasiou +Date: Thu Dec 20 14:53:36 2018 +0200 + + [ML] Reduce persistent tasks periodic reassignment interval in ... (#36845) + + ... MlDistributedFailureIT.testLoseDedicatedMasterNode. + + An intermittent failure has been observed in + `MlDistributedFailureIT. testLoseDedicatedMasterNode`. + The test launches a cluster comprised by a dedicated master node + and a data and ML node. It creates a job and datafeed and starts them. + It then shuts down and restarts the master node. Finally, the test asserts + that the two tasks have been reassigned within 10s. + + The intermittent failure is due to the assertions that the tasks have been + reassigned failing. Investigating the failure revealed that the `assertBusy` + that performs that assertion times out. Furthermore, it appears that the + job task is not reassigned because the memory tracking info is stale. + + Memory tracking info is refreshed asynchronously when a job is attempted + to be reassigned. Tasks are attempted to be reassigned either due to a relevant + cluster state change or periodically. The periodic interval is controlled by a cluster + setting called `cluster.persistent_tasks.allocation.recheck_interval` and defaults to 30s. + + What seems to be happening in this test is that if all cluster state changes after the + master node is restarted come through before the async memory info refresh completes, + then the job might take up to 30s until it is attempted to reassigned. Thus the `assertBusy` + times out. + + This commit changes the test to reduce the periodic check that reassigns persistent + tasks to `200ms`. If the above theory is correct, this should eradicate those failures. + + Closes #36760 + +commit 0f2f00a20a6eae5b1fa9931f3b300746eb28e1c6 +Author: David Roberts +Date: Thu Dec 20 12:49:57 2018 +0000 + + [ML] Resolve 7.0.0 TODOs in ML code (#36842) + + This change cleans up a number of ugly BWC + workarounds in the ML code. + + 7.0 cannot run in a mixed version cluster with + versions prior to 6.7, so code that deals with + these old versions is no longer required. + + Closes #29963 + +commit 1236461e3e94a785136c467471bc73591dfd10c5 +Author: Andrei Stefan +Date: Thu Dec 20 13:39:55 2018 +0200 + + SQL: Make sure now() always uses milliseconds precision (#36877) + + * This change is to account for different system clock implementations + or different Java versions (for Java 8, milliseconds precision is used; + for Java 9+ a system specific clock implementation is used which can + have greater precision than what we need here). + +commit 9f75b6ea58103c6dd3d8f73941caf79fadd83514 +Author: Alpar Torok +Date: Thu Dec 20 12:15:26 2018 +0200 + + Fix build on sles with new docker version (#36707) + + The build hash for some Docker versions is + longer than 7 characters. + + Closes #36414 + +commit 9c2980a55e11266ef419fec027b20025175f0647 +Author: Martijn van Groningen +Date: Thu Dec 20 09:58:04 2018 +0100 + + Remote cluster license checker and no license info. (#36837) + + Fail with a descriptive error when the xpack info returns no license info. + + Relates to #36815 + +commit cfc0a47232a7c567d4a2d07b159d1a0a8021e7de +Author: Ryan Ernst +Date: Thu Dec 20 00:17:06 2018 -0800 + + Core: Deprecate negative epoch timestamps (#36793) + + Negative timestamps are currently supported in joda time. These are + dates before epoch. However, it doesn't really make sense to have a + negative timestamp, since this is a modern format. Any dates before + epoch can be represented with normal date formats, like ISO8601. + Additionally, implementing negative epoch timestamp parsing in java time + has an edge case which would more than double the code required. This + commit deprecates use of negative epoch timestamps. + +commit 6221d6be499ee7d2eb23810733897038ad265623 +Author: Marios Trivyzas +Date: Thu Dec 20 09:56:07 2018 +0200 + + SQL: Fix issue with always false filter involving functions (#36830) + + When a filter is evaluated to false then it becomes a LocalRelation + with an EmptyExecutable. The LocalRelation in turn, becomes a + LocalExec and the the SkipQueryIfFoldingProjection was wrongly + converting it to a SingletonExecutable. Moreover made a change, so + that the queries without FROM clause, which are supposed to return a + single row, to become a LocalRelation with a SingletonExecutable + instead of EmptyExecutable to avoid mixing up with the ones operating + on a table but with a filter that evaluates to false. + + Fixes: #35980 + +commit a7f344cc7f5ab5327f3acf9118e6a577bd78c4f3 +Author: Tim Brooks +Date: Wed Dec 19 16:58:43 2018 -0700 + + Use ByteBuffer#slice in BytesReference wrapper (#36862) + + Currently the ByteBufferReference does not duplicate the buffer. + This means that any changes to the buffer's limit or position will + impact the reference. This can lead to unexpected behavior. This commit + uses the ByteBuffer#slice method to ensure that the reference retains + its own ByteBuffer. + +commit 8b8121279aeeac823dfd5786e6b3f6afde81e784 +Author: lcawl +Date: Wed Dec 19 15:28:25 2018 -0800 + + [DOCS] Fix typos + +commit ecb822c6664357086756d0d28fb6a4fb44b7e3b1 +Author: Julie Tibshirani +Date: Wed Dec 19 15:20:20 2018 -0800 + + Deprecate the document create endpoint. (#36863) + +commit c4e12f06d7063a97671f634b3e52a56bc04c478e +Author: Ioannis Kakavas +Date: Thu Dec 20 01:11:55 2018 +0200 + + Link to the SAML troubleshooting doc from guide (#36754) + + Explicitly call out the existence of the troubleshooting guide so + that hopefully users can solve common and easy problems with their + initial configuration + +commit 4140b9eede532cebc6b67ebc90e7ef42e64a5dad +Author: Lisa Cawley +Date: Wed Dec 19 14:53:37 2018 -0800 + + [DOCS] Update X-Pack terminology in security docs (#36564) + +commit 9c1e47d434fbd59cee49a6ced14b698d9766933e +Author: Mayya Sharipova +Date: Wed Dec 19 17:02:14 2018 -0500 + + Add 6.5.5 version constants + +commit 32ef80f3d4bb8086a954b14be0baf80a0979620e +Author: Julie Tibshirani +Date: Wed Dec 19 12:59:25 2018 -0800 + + Avoid duplicate types deprecation messages in search-related APIs. (#36802) + +commit 3412627efe56afb118179dcb5a42d6a3c86b2a52 +Author: David Causse +Date: Wed Dec 19 21:42:18 2018 +0100 + + QueryRescorer should keep the window size when rewriting (#36836) + + This attribute being controlled by the parent class it's easy to miss + it during rewrites. + +commit 5185d61ad190e2bebaf05cc8aa98ddfceb2c1309 +Author: Julie Tibshirani +Date: Wed Dec 19 11:25:45 2018 -0800 + + Make sure to use _doc for the type in SQL REST tests. (#36806) + +commit c85c16bd94f92309cccdc120881b399a12f5838e +Author: Ryan Ernst +Date: Wed Dec 19 11:20:35 2018 -0800 + + Core: Revert back to joda's multi date formatters (#36814) + + This commit partially reverts #36447 by using the ability of Joda time's + DateTimeFormatterBuilder to append multiple parsers instead of using the + MergedDateFormatter. The MergedDateFormatter will be removed in a future + change, as it is not as performant due to creating potentially many + exceptions during heavy date parsing. This change is a stop-gap until + that followup is ready. + + closes #36602 + +commit d39956c65c07055179d1a305b19ce545671d1c6d +Author: Gordon Brown +Date: Wed Dec 19 12:11:30 2018 -0700 + + Remove `indexing_complete` when removing policy (#36620) + + Leaving `index.lifecycle.indexing_complete` in place when removing the + lifecycle policy from an index can cause confusion, as if a new policy + is associated with the policy, rollover will be silently skipped. + Removing that setting when removing the policy from an index makes + associating a new policy with the index more involved, but allows ILM to + fail loudly, rather than silently skipping operations which the user may + assume are being performed. + + * Adjust order of checks in WaitForRolloverReadyStep + + This allows ILM to error out properly for indices that have a valid + alias, but are not the write index, while still handling + `indexing_complete` on old-style aliases and rollover (that is, those + which only point to a single index at a time with no explicit write + index) + +commit f99975bc297db18fa716b9b23cc99561cf17239b +Author: Gordon Brown +Date: Wed Dec 19 12:09:37 2018 -0700 + + Rename confusing variable holding name of Logstash index template (#36840) + + The previous name was confusing, so this commit clarifies it. + +commit 7d35cd48394f4afc4b778e8c7265c6689992c3eb +Author: lcawl +Date: Wed Dec 19 10:26:45 2018 -0800 + + [DOCS] Edits release note title + +commit b65ce47e06622d842ca98671513ababfff3b633a +Author: Tom Callahan +Date: Wed Dec 19 12:48:19 2018 -0500 + + [DOCS] Add PR list for 7.0.0-alpha2 (#36803) + + Adds the PR list for 7.0.0-alpha2 + +commit 9058698d9dd9f581feb6ac3d58322d435e9689f3 +Author: Christoph Büscher +Date: Wed Dec 19 18:45:43 2018 +0100 + + Removing some deprecated methods (#36829) + + Changes: + * Removed deprecated method in InnerHitBuilder + * Removed fields() from SearchRequestBuilder + * Removed deprecated GeoDistanceSortBuilder#geohashes + +commit de052ae533a84ecc0db37fa1a224ea9fa5757526 +Author: Christoph Büscher +Date: Wed Dec 19 18:19:16 2018 +0100 + + Bump laster 6.x version in VersionTests (#36851) + +commit b63f9b967c544c972ff674e22eb671b98c966c7e +Author: Nhat Nguyen +Date: Wed Dec 19 11:19:25 2018 -0500 + + Fix translog bwc serialization (#36676) + + Serializing of a Translog#Index from v7.0.0 to 6.x is broken since + #29224 where we removed the _parent field. + + Relates #29224 + +commit 9b1534b79b4c7d84a0260d95dc179012cb1630b6 +Author: Nicholas Knize +Date: Wed Dec 19 10:10:49 2018 -0600 + + Update breaking changes doc with changes to geo_shape defaults + + This commit updates the breaking changes doc to highlight the + impact of changing geo_shape defaults on timed index creation. + +commit 8f736c3ce9eb7d9f1daa27564e4b73df5d433495 +Author: lcawl +Date: Wed Dec 19 08:01:41 2018 -0800 + + [DOCS] Fixes broken link to profiling aggregations + +commit 667c06dc839639e36ed9513c2ff5098fe159aef1 +Author: Jim Ferenczi +Date: Wed Dec 19 22:18:53 2018 +0700 + + Add link to script score query in the top level docs (#36416) + + * add link to script score query in the top level docs + + * Correct references in script-score-query.asciidoc + +commit fb24469fe7526e15053d0fd3d92d4391513ea6df +Merge: c99fd6a53b8 9584adf9d95 +Author: Tanguy Leroux +Date: Wed Dec 19 16:17:26 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 9584adf9d9579f59c0e98ee4d5f2cf54d55be2e7 +Author: Costin Leau +Date: Wed Dec 19 17:14:01 2018 +0200 + + SQL: Enhance Verifier to prevent aggregate or grouping functions from (#36799) + + Improve Verifier to prevent aggregate or grouping functions from + being used in a WHERE clause. + + Fix #36798 + +commit d31eaf731393fafc35beac4867edb2f562a4f816 +Author: Andrei Stefan +Date: Wed Dec 19 16:36:16 2018 +0200 + + SQL: protocol returns ISO 8601 String formatted dates instead of Long for JDBC/ODBC requests (#36800) + + * Change the way the protocol returns date fields from Long values in case + of JDBC/ODBC, to ISO 8601 with millis String. + +commit a34a3532ce135691f858a09c11206f73c8ed4da2 +Author: Ignacio Vera +Date: Wed Dec 19 15:29:09 2018 +0100 + + [Javadoc]: Remove lucene tags (#36834) + + Remove lucene tags as they break gradle javadoc task + + Relates #36794 + +commit d43cbdab9793587daf30256fd33ba4f2a1d6b7d2 +Author: David Kyle +Date: Wed Dec 19 13:43:43 2018 +0000 + + [ML] ensure the ml-config index (#36792) (#36832) + +commit f2a5373495f38aa9148c811732cf858eb5c19eeb +Author: Jason Tedor +Date: Wed Dec 19 08:24:10 2018 -0500 + + Add the 6.7.0 constant to the master branch + + Now that the 6.x branch has been bumped to the 6.7.0 version, this + commit adds knowledge of the 6.7.0 version to the master branch. + +commit ad20d6bb836bb7a1c6a0438c36eb90de4bb92815 +Author: David Roberts +Date: Wed Dec 19 13:06:24 2018 +0000 + + [ML] Followup to annotations index creation (#36824) + + Fixes two minor problems reported after merge of #36731: + + 1. Name the creation method to make clear it only creates + if necessary + 2. Avoid multiple simultaneous in-flight creation requests + +commit 8f141b8a41554f07217e1eaa6c0028d21c833ae4 +Author: Yannick Welsch +Date: Wed Dec 19 13:59:58 2018 +0100 + + Fix ClusterInfoServiceIT timeouts (#36758) + + The test testClusterInfoServiceInformationClearOnError relies on timing behavior. It sets + InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING to 1s and relies on the + fact that the stats request completes within that timeframe (which our ever-so-slow CI seems to + violate at times). Unfortunately the logging has been misimplemented in InternalClusterInfoService, + so the corresponding log messages showing that the requests have timed out are missing for this. + The issue can be locally reproduced by reducing the timeout to something lower. + + Closes #36554 + +commit 18691daebe904ead72cb30e92f257026da92a6a8 +Author: Martijn van Groningen +Date: Wed Dec 19 13:56:54 2018 +0100 + + [TEST] Renamed ccr qa module. + +commit 63aa8756b26541248a968d84c229a585f18fc1b1 +Author: Albert Zaharovits +Date: Wed Dec 19 14:56:40 2018 +0200 + + Add X-Forwarded-For to the logfile audit (#36427) + + Extracts the value of the X-Forwarded-For HTTP request header and + places it in the audit entries from the logfile output. + +commit 3cc0cf03c6da31304e5dca27a8a672e5159c777e +Author: Martijn van Groningen +Date: Wed Dec 19 13:51:12 2018 +0100 + + [TEST] No need to specifically check licensesMetaData on master node. + +commit 3f8f907606e3b7c5f5d370c36da58876bcbd0051 +Author: David Turner +Date: Wed Dec 19 12:51:14 2018 +0000 + + Extend time allowed to detect disconnection (#36827) + + If the master sends both its follower checks just before disconnection then + neither will receive a response, meaning it must wait for the checks to time + out and send another in order to detect the disconnection and stand down. + + Closes #36788 + +commit 487a1c4f71262123136dd0f5e6a5cbf0f2696399 +Author: Yannick Welsch +Date: Wed Dec 19 13:26:04 2018 +0100 + + Fix cluster state persistence for single-node discovery (#36825) + + Single-node discovery is not persisting cluster states, which was caused by a recent 7.0-only + refactoring. This commit ensures that the cluster state is properly persisted when using single-node + discovery and adds a corresponding test. + +commit 216b15410768694019240636c08180245328ac3a +Author: Boaz Leskes +Date: Wed Dec 19 13:15:05 2018 +0100 + + Replace 0L with an UNASSIGNED_PRIMARY_TERM constant (#36819) + + * Replace 0L with an UNASSIGNED_PRIMARY_TERM constant + + 0 is an illegal value for a primary term that is often used to indicate + the primary term isn't assigned or is irrelevant. This PR replaces the + usage of 0 with a constant, to improve readability and so it can be + tracked and if needed, replaced. + + * feedback + +commit 978713a67c5cece341c3f94406f8864574c42c22 +Author: Armin Braun +Date: Wed Dec 19 13:14:36 2018 +0100 + + SNAPSHOTS+TESTS: Correctly Wait for Clean State (#36801) + + * Test must wait until there is no in-progress deletion as well here since the master failover leads to a snapshot deletion + * Closes #36779 + +commit a6af33ef0b1c59cf0152634957d900b8dd0adb9c +Author: Martijn van Groningen +Date: Wed Dec 19 13:03:35 2018 +0100 + + [TEST] Wait for license metadata to be installed + +commit 344917efabe9b9bd5df0cd70a7be85d417d4964a +Author: Alan Woodward +Date: Wed Dec 19 11:12:18 2018 +0000 + + Add script filter to intervals (#36776) + + This commit adds the ability to filter out intervals based on their start and end position, and internal + gaps: + ``` + POST _search + { + "query": { + "intervals" : { + "my_text" : { + "match" : { + "query" : "hot porridge", + "filter" : { + "script" : { + "source" : "interval.start > 10 && interval.end < 20 && interval.gaps == 0" + } + } + } + } + } + } + } + ``` + +commit 1345dff5076a57981df40cea01b4c3a7f33ea08b +Author: Przemyslaw Gomulka +Date: Wed Dec 19 11:29:21 2018 +0100 + + Fix line length in org.elasticsearch.snapshots (#36646) + + Remove the line length suppression for this package and fix offending + lines in both main and test + + relates #34884 + +commit d2ce576c8cfd43aedabd0bf5c695d8745156c69f +Author: Luca Cavanna +Date: Wed Dec 19 10:47:17 2018 +0100 + + Use SearchRequest copy constructor in ExpandSearchPhase (#36772) + + Relates to #36641 + +commit dd540ef6180b8838c79aad31efc69e4ad1e2b5f8 +Author: Alan Woodward +Date: Wed Dec 19 08:55:05 2018 +0000 + + Use index-prefix fields for terms of length min_chars - 1 (#36703) + + The default index_prefix settings will index prefixes of between 2 and 5 characters in length. + Currently, if a prefix search falls outside of this range at either end we fall back to a standard prefix + expansion, which is still very expensive for single character prefixes. However, we have an option + here to use a wildcard expansion rather than a prefix expansion, so that a query of a* gets remapped + to a? against the _index_prefix field - likely to be a very small set of terms, and certain to be much + smaller than a* against the whole index. + + This commit adds this extra level of mapping for any prefix term whose length is one less than + the min_chars parameter of the index_prefixes field. + +commit 132ccbec2fdeebe77009d26cc93bd0709b7b8479 +Author: Christoph Büscher +Date: Wed Dec 19 09:49:23 2018 +0100 + + [Docs] Extend common-grams-tokenfilter doctest example (#36807) + + Adding an example output using the "_analyze" API and expected response. + +commit c99fd6a53b805bb25e22ee8e769b4ffba19b56fb +Merge: 103c4d4762f 1e9d0bb01e1 +Author: Tanguy Leroux +Date: Wed Dec 19 09:34:59 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 1e9d0bb01e17d4ad6f5394007e9214595f4dcd4e +Author: Nhat Nguyen +Date: Wed Dec 19 03:12:34 2018 -0500 + + AwaitsFix testRebuildLocalCheckpointTracker + + Tracked at #36470 + +commit 8e5db90eec696110bb874e0ed0c0b2cff6412383 +Author: Simon Willnauer +Date: Wed Dec 19 07:39:10 2018 +0100 + + Never corrupt fully deleted segments in tests (#36741) + + Today we might corrupt a fully deleted segment which is then pruned + once a snapshot is taken. This causes random test failures in CorruptedFileIT. + This change hardens the selection of files to corrupt and removes some fragile + code preventing fully deleted segments to be taken into account. + + Closes #36526 + +commit e9ef5bdce83f466b019a59f81186ecee0dd4bd29 +Author: Alpar Torok +Date: Wed Dec 19 08:25:20 2018 +0200 + + Converting randomized testing to create a separate unitTest task instead of replacing the builtin test task (#36311) + + - Create a separate unitTest task instead of Gradle's built in + - convert all configuration to use the new task + - the built in task is now disabled + +commit 8015560bd4c02b11e0c92f60d87b875d8bc6b1b5 +Author: Alpar Torok +Date: Wed Dec 19 08:06:48 2018 +0200 + + Remove line length supressions from plugins (#36708) + +commit 06b175dd6973c4b94e72b2864a19ff4aad4e5ea2 +Author: Tim Vernum +Date: Wed Dec 19 17:02:26 2018 +1100 + + [TEST] Mute MlMigrationFullClusterRestartIT + + AwaitsFix: https://github.com/elastic/elasticsearch/issues/36816 + + This test fails quite reliably. + +commit c65d2339db994db9e1d9bab98abbbf663bf1b8df +Author: Jason Tedor +Date: Tue Dec 18 22:20:51 2018 -0500 + + Simplify formatting in geo-IP test case + + This commit cleans up the formatting in a geo-IP processor factory test + case so that the code does not consume a silly number of lines. + +commit 273b37aaa43926c24a49539ae875f43c9cb4b71a +Author: Jason Tedor +Date: Tue Dec 18 22:15:33 2018 -0500 + + Make the ingest-geoip databases even lazier to load (#36679) + + Today we try to load the ingest-geoip databases lazily. Currently they + are loaded as soon as any pipeline that uses an ingest-geoip processor + is created. This is not lazy enough. For example, we could only load the + databases the first time that they are actually used. This would ensure + that we load the minimal set of data to support an in-use pipeline + (instead of *all* of the data). This can come up in a couple of + ways. One is when a subset of the database is used (e.g., the city + database versus the country database versus the ASN database). Another + is when the plugins are installed on non-ingest nodes (e.g., master-only + nodes); we would never use the database in this case yet they are + currently being loaded occupying ~60 MB of the heap. This commit makes + the ingest-geoip databases as lazy as possible. + + Co-authored-by: Martijn van Groningen + +commit aaf466ff5e6d109f491a40aeb1b23d623b50108e +Author: Tim Brooks +Date: Tue Dec 18 19:01:13 2018 -0700 + + Revert transport.port change for tests (#36809) + + Commit #36786 updated docs and strings to reference transport.port instead of + transport.tcp.port. However, this breaks backwards compatibility tests + as the tests rely on string configurations and transport.port does not + exist prior to 6.6. This commit reverts the places were we reference + transport.tcp.port for tests. This work will need to be reintroduced in + a backwards compatible way. + +commit 75f1c79d9f00bbef2dd1b8570c7eece53cfb3888 +Author: Benjamin Trent +Date: Tue Dec 18 19:14:18 2018 -0600 + + Adding more docs for delayed data detection (#36738) + + * Adding more docs for delayed data detection + +commit 1d429cf1c9a551ec008d80798fc16273e676c48f +Author: Benjamin Trent +Date: Tue Dec 18 18:40:38 2018 -0600 + + ML having delayed data detection create annotations (#36796) + + * ML having delayed data detection create annotations + + * adding upsertAsDoc, audit, and changing user + + * changing update to just index the doc with the id set + +commit 504cfb2fb1ec3c5f6e9dc8b3b1d638414690faff +Author: lcawl +Date: Tue Dec 18 15:20:19 2018 -0800 + + [DOCS] Adds missing anchors for profile API + +commit 20b58f0b0ffdc8e1f0bd7c1fe99b6b5ee1c40af6 +Author: Nick Knize +Date: Tue Dec 18 16:48:30 2018 -0600 + + [GEO] Fork Lucene's LatLonShape Classes to local lucene package (#36794) + + Lucene 7.6 uses a smaller encoding for LatLonShape. This commit forks the LatLonShape classes to Elasticsearch's local lucene package. These classes will be removed on the release of Lucene 7.6. + +commit a8387592dbcb4d180a043d773c052ae407712dab +Author: lcawl +Date: Tue Dec 18 14:43:29 2018 -0800 + + [DOCS] Fixes broken links to tcp_tranport_profiles + +commit 7bd7f022ef70dce12a7a32e4b118e2cc08528784 +Author: Gordon Brown +Date: Tue Dec 18 15:37:54 2018 -0700 + + Rename logstash management index template (#36626) + + The logstash management template was named in such a way as to confuse + users, who misunderstood it to be a template for indices created by + logstash. It is now renamed to more clearly communicate its purpose and + match the format of the other templates for system indices. + +commit 6d49873ab7ecd73416564a63a550d34b54856218 +Author: Zachary Tong +Date: Tue Dec 18 16:35:39 2018 -0500 + + Fix Rollup's metadata parser (#36791) + + The parser used for rollup configs in _meta fields was not able to + handle unrelated data in the meta field. If an unrelated object + was encountered, it would half-consume the JSON object, realize it + wasn'ta rollup config, then stop parsing. This would leave the object + halfway consumed and the parsing framework would throw an exception. + + This commit replaces the parsing logic with a set of minimal parsers, + each for the specific component we care about (`_doc`, `_meta`, + `_rollup`) and configured to ignore unknown fields where applicable. + + More verbose, but less hacky than before and should be more robust. + + Also adds tests (randomized and explicit) to make sure this doesn't + break in the future. + +commit 47a9a8de4952b70dd0ead40d9bae998a1bb95f38 +Author: Tim Brooks +Date: Tue Dec 18 13:09:58 2018 -0700 + + Update transport docs and settings for changes (#36786) + + This is related to #36652. In 7.0 we plan to deprecate a number of + settings that make reference to the concept of a tcp transport. We + mostly just have a single transport type now (based on tcp). Settings + should only reference tcp if they are referring to socket options. This + commit updates the settings in the docs. And removes string usages of + the old settings. Additionally it adds a missing remote compress setting + to the docs. + +commit 10f64088143ab3b6169fac63e2470084c4082fb2 +Author: Boaz Leskes +Date: Tue Dec 18 20:03:07 2018 +0100 + + enable bwc tests and switch transport serialization version to 6.6.0 for CAS features + + Relates to #36148 + +commit c3c785cc1e157f1226ae5290c8226733ae321bdc +Author: Lisa Cawley +Date: Tue Dec 18 10:46:03 2018 -0800 + + [DOCs] Adds ml-cpp PRs to alpha release notes (#36790) + +commit e35de2ea2c9a108d5aa60b72ac72f70ec8e8a398 +Author: Yannick Welsch +Date: Tue Dec 18 19:23:35 2018 +0100 + + Synchronize WriteReplicaResult callbacks (#36770) + + TransportWriteAction.WriteReplicaResult is not properly synchronized, which can lead to a data race + between the thread that calls respond and the AsyncAfterWriteAction that calls either onSuccess or + onFailure. This data race results in the response listener not being called, which ultimately results in + a stuck replication task on the replica. + +commit 1fa105658ef1a1c8dffd8ca4336c2da0f1683656 +Author: Tim Brooks +Date: Tue Dec 18 11:23:13 2018 -0700 + + Add CcrRestoreSourceService to track sessions (#36578) + + This commit is related to #36127. It adds a CcrRestoreSourceService to + track Engine.IndexCommitRef need for in-process file restores. When a + follower starts restoring a shard through the CcrRepository it opens a + session with the leader through the PutCcrRestoreSessionAction. The + leader responds to the request by telling the follower what files it + needs to fetch for a restore. This is not yet implemented. + + Once, the restore is complete, the follower closes the session with the + DeleteCcrRestoreSessionAction action. + +commit 7de85f55e3add826faaff8bed32d0a0d74134f5b +Author: Jack Conradson +Date: Tue Dec 18 10:14:48 2018 -0800 + + [Painless] Add tests for boxed return types (#36747) + + Adds tests for each primitive/boxed and def type to be implicitly cast to an + appropriate boxed return type from a method. + +commit 8ec8342a521e2ffc1ae3a7debd788551d8e5fdb8 +Author: Ryan Ernst +Date: Tue Dec 18 10:05:27 2018 -0800 + + Internal: Remove originalSettings from Node (#36569) + + This commit removes the originalSettings member from Node. It was only + needed to allows test clusters to recreate the node in certain + situations. Instead, the test cluster now keeps track of these settings. + +commit 32ce1240e4cec02079b16d52f63bfdae67d39ddd +Author: Tal Levy +Date: Tue Dec 18 09:53:25 2018 -0800 + + [ILM][DOCS] Update ILM API authorization docs (#36749) + + the `view_index_metadata` privilege. Now the ILM API authorization + documentation mentions the need for these roles in more specific + ways. + +commit 0b22ca3a0f911bdb60023cc88003838c5ebeabfb +Author: Ryan Ernst +Date: Tue Dec 18 09:51:03 2018 -0800 + + Core: Deprecate use of scientific notation in epoch time parsing (#36691) + + The joda epoch parsing code currently supports passing epoch time as a + number in scientific notation. However, no systems appear to exist which + output timestamps in scientific notation. In java time, it is + particularly complex to implement scientific notation timestamp parsing + within a DateTimeFormatter. This commit adds a deprecation warning when + the epoch time parsers in joda parse scientific notation, so that it can + be removed when switching to java time. + joda are passed a time in scientific notation. + +commit e294056bbf3ce579c41847d0bbe4a407f9f3f1cd +Author: David Kyle +Date: Tue Dec 18 17:45:31 2018 +0000 + + [ML] Merge the Jindex master feature branch (#36702) + + * [ML] Job and datafeed mappings with index template (#32719) + + Index mappings for the configuration documents + + * [ML] Job config document CRUD operations (#32738) + + * [ML] Datafeed config CRUD operations (#32854) + + * [ML] Change JobManager to work with Job config in index (#33064) + + * [ML] Change Datafeed actions to read config from the config index (#33273) + + * [ML] Allocate jobs based on JobParams rather than cluster state config (#33994) + + * [ML] Return missing job error when .ml-config is does not exist (#34177) + + * [ML] Close job in index (#34217) + + * [ML] Adjust finalize job action to work with documents (#34226) + + * [ML] Job in index: Datafeed node selector (#34218) + + * [ML] Job in Index: Stop and preview datafeed (#34605) + + * [ML] Delete job document (#34595) + + * [ML] Convert job data remover to work with index configs (#34532) + + * [ML] Job in index: Get datafeed and job stats from index (#34645) + + * [ML] Job in Index: Convert get calendar events to index docs (#34710) + + * [ML] Job in index: delete filter action (#34642) + + This changes the delete filter action to search + for jobs using the filter to be deleted in the index + rather than the cluster state. + + * [ML] Job in Index: Enable integ tests (#34851) + + Enables the ml integration tests excluding the rolling upgrade tests and a lot of fixes to + make the tests pass again. + + * [ML] Reimplement established model memory (#35500) + + This is the 7.0 implementation of a master node service to + keep track of the native process memory requirement of each ML + job with an associated native process. + + The new ML memory tracker service works when the whole cluster + is upgraded to at least version 6.6. For mixed version clusters + the old mechanism of established model memory stored on the job + in cluster state was used. This means that the old (and complex) + code to keep established model memory up to date on the job object + has been removed in 7.0. + + Forward port of #35263 + + * [ML] Need to wait for shards to replicate in distributed test (#35541) + + Because the cluster was expanded from 1 node to 3 indices would + initially start off with 0 replicas. If the original node was + killed before auto-expansion to 1 replica was complete then + the test would fail because the indices would be unavailable. + + * [ML] DelayedDataCheckConfig index mappings (#35646) + + * [ML] JIndex: Restore finalize job action (#35939) + + * [ML] Replace Version.CURRENT in streaming functions (#36118) + + * [ML] Use 'anomaly-detector' in job config doc name (#36254) + + * [ML] Job In Index: Migrate config from the clusterstate (#35834) + + Migrate ML configuration from clusterstate to index for closed jobs + only once all nodes are v6.6.0 or higher + + * [ML] Check groups against job Ids on update (#36317) + + * [ML] Adapt to periodic persistent task refresh (#36633) + + * [ML] Adapt to periodic persistent task refresh + + If https://github.com/elastic/elasticsearch/pull/36069/files is + merged then the approach for reallocating ML persistent tasks + after refreshing job memory requirements can be simplified. + This change begins the simplification process. + + * Remove AwaitsFix and implement TODO + + * [ML] Default search size for configs + + * Fix TooManyJobsIT.testMultipleNodes + + Two problems: + + 1. Stack overflow during async iteration when lots of + jobs on same machine + 2. Not effectively setting search size in all cases + + * Use execute() instead of submit() in MlMemoryTracker + + We don't need a Future to wait for completion + + * [ML][TEST] Fix NPE in JobManagerTests + + * [ML] JIindex: Limit the size of bulk migrations (#36481) + + * [ML] Prevent updates and upgrade tests (#36649) + + * [FEATURE][ML] Add cluster setting that enables/disables config migration (#36700) + + This commit adds a cluster settings called `xpack.ml.enable_config_migration`. + The setting is `true` by default. When set to `false`, no config migration will + be attempted and non-migrated resources (e.g. jobs, datafeeds) will be able + to be updated normally. + + Relates #32905 + + * [ML] Snapshot ml configs before migrating (#36645) + + * [FEATURE][ML] Split in batches and migrate all jobs and datafeeds (#36716) + + Relates #32905 + + * SQL: Fix translation of LIKE/RLIKE keywords (#36672) + + * SQL: Fix translation of LIKE/RLIKE keywords + + Refactor Like/RLike functions to simplify internals and improve query + translation when chained or within a script context. + + Fix #36039 + Fix #36584 + + * Fixing line length for EnvironmentTests and RecoveryTests (#36657) + + Relates #34884 + + * Add back one line removed by mistake regarding java version check and + COMPAT jvm parameter existence + + * Do not resolve addresses in remote connection info (#36671) + + The remote connection info API leads to resolving addresses of seed + nodes when invoked. This is problematic because if a hostname fails to + resolve, we would not display any remote connection info. Yet, a + hostname not resolving can happen across remote clusters, especially in + the modern world of cloud services with dynamically chaning + IPs. Instead, the remote connection info API should be providing the + configured seed nodes. This commit changes the remote connection info to + display the configured seed nodes, avoiding a hostname resolution. Note + that care was taken to preserve backwards compatibility with previous + versions that expect the remote connection info to serialize a transport + address instead of a string representing the hostname. + + * [Painless] Add boxed type to boxed type casts for method/return (#36571) + + This adds implicit boxed type to boxed types casts for non-def types to create asymmetric casting relative to the def type when calling methods or returning values. This means that a user calling a method taking an Integer can call it with a Byte, Short, etc. legally which matches the way def works. This creates consistency in the casting model that did not previously exist. + + * SNAPSHOTS: Adjust BwC Versions in Restore Logic (#36718) + + * Re-enables bwc tests with adjusted version conditions now that #36397 enables concurrent snapshots in 6.6+ + + * ingest: fix on_failure with Drop processor (#36686) + + This commit allows a document to be dropped when a Drop processor + is used in the on_failure fork of the processor chain. + + Fixes #36151 + + * Initialize startup `CcrRepositories` (#36730) + + Currently, the CcrRepositoryManger only listens for settings updates + and installs new repositories. It does not install the repositories that + are in the initial settings. This commit, modifies the manager to + install the initial repositories. Additionally, it modifies the ccr + integration test to configure the remote leader node at startup, instead + of using a settings update. + + * [TEST] fix float comparison in RandomObjects#getExpectedParsedValue + + This commit fixes a test bug introduced with #36597. This caused some + test failure as stored field values comparisons would not work when CBOR + xcontent type was used. + + Closes #29080 + + * [Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320) + + This commit exposes lucene's LatLonShape field as the + default type in GeoShapeFieldMapper. To use the new + indexing approach, simply set "type" : "geo_shape" in + the mappings without setting any of the strategy, precision, + tree_levels, or distance_error_pct parameters. Note the + following when using the new indexing approach: + + * geo_shape query does not support querying by + MULTIPOINT. + * LINESTRING and MULTILINESTRING queries do not + yet support WITHIN relation. + * CONTAINS relation is not yet supported. + The tree, precision, tree_levels, distance_error_pct, + and points_only parameters are deprecated. + + * TESTS:Debug Log. IndexStatsIT#testFilterCacheStats + + * ingest: support default pipelines + bulk upserts (#36618) + + This commit adds support to enable bulk upserts to use an index's + default pipeline. Bulk upsert, doc_as_upsert, and script_as_upsert + are all supported. + + However, bulk script_as_upsert has slightly surprising behavior since + the pipeline is executed _before_ the script is evaluated. This means + that the pipeline only has access the data found in the upsert field + of the script_as_upsert. The non-bulk script_as_upsert (existing behavior) + runs the pipeline _after_ the script is executed. This commit + does _not_ attempt to consolidate the bulk and non-bulk behavior for + script_as_upsert. + + This commit also adds additional testing for the non-bulk behavior, + which remains unchanged with this commit. + + fixes #36219 + + * Fix duplicate phrase in shrink/split error message (#36734) + + This commit removes a duplicate "must be a" from the shrink/split error + messages. + + * Deprecate types in get_source and exist_source (#36426) + + This change adds a new untyped endpoint `{index}/_source/{id}` for both the + GET and the HEAD methods to get the source of a document or check for its + existance. It also adds deprecation warnings to RestGetSourceAction that emit + a warning when the old deprecated "type" parameter is still used. Also updating + documentation and tests where appropriate. + + Relates to #35190 + + * Revert "[Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320)" + + This reverts commit 5bc7822562a6eefa4a64743233160cdc9f431adf. + + * Enhance Invalidate Token API (#35388) + + This change: + + - Adds functionality to invalidate all (refresh+access) tokens for all users of a realm + - Adds functionality to invalidate all (refresh+access)tokens for a user in all realms + - Adds functionality to invalidate all (refresh+access) tokens for a user in a specific realm + - Changes the response format for the invalidate token API to contain information about the + number of the invalidated tokens and possible errors that were encountered. + - Updates the API Documentation + + After back-porting to 6.x, the `created` field will be removed from master as a field in the + response + + Resolves: #35115 + Relates: #34556 + + * Add raw sort values to SearchSortValues transport serialization (#36617) + + In order for CCS alternate execution mode (see #32125) to be able to do the final reduction step on the CCS coordinating node, we need to serialize additional info in the transport layer as part of each `SearchHit`. Sort values are already present but they are formatted according to the provided `DocValueFormat` provided. The CCS node needs to be able to reconstruct the lucene `FieldDoc` to include in the `TopFieldDocs` and `CollapseTopFieldDocs` which will feed the `mergeTopDocs` method used to reduce multiple search responses (one per cluster) into one. + + This commit adds such information to the `SearchSortValues` and exposes it through a new getter method added to `SearchHit` for retrieval. This info is only serialized at transport and never printed out at REST. + + * Watcher: Ensure all internal search requests count hits (#36697) + + In previous commits only the stored toXContent version of a search + request was using the old format. However an executed search request was + already disabling hit counts. In 7.0 hit counts will stay enabled by + default to allow for proper migration. + + Closes #36177 + + * [TEST] Ensure shard follow tasks have really stopped. + + Relates to #36696 + + * Ensure MapperService#getAllMetaFields elements order is deterministic (#36739) + + MapperService#getAllMetaFields returns an array, which is created out of + an `ObjectHashSet`. Such set does not guarantee deterministic hash + ordering. The array returned by its toArray may be sorted differently + at each run. This caused some repeatability issues in our tests (see #29080) + as we pick random fields from the array of possible metadata fields, + but that won't be repeatable if the input array is sorted differently at + every run. Once setting the tests seed, hppc picks that up and the sorting is + deterministic, but failures don't repeat with the seed that gets printed out + originally (as a seed was not originally set). + See also https://issues.carrot2.org/projects/HPPC/issues/HPPC-173. + + With this commit, we simply create a static sorted array that is used for + `getAllMetaFields`. The change is in production code but really affects + only testing as the only production usage of this method was to iterate + through all values when parsing fields in the high-level REST client code. + Anyways, this seems like a good change as returning an array would imply + that it's deterministically sorted. + + * Expose Sequence Number based Optimistic Concurrency Control in the rest layer (#36721) + + Relates #36148 + Relates #10708 + + * [ML] Mute MlDistributedFailureIT + +commit ea9b08dee127c7434aa308faf7c801f3e4e86315 +Author: Alexander Reelsen +Date: Tue Dec 18 18:25:06 2018 +0100 + + Tests: Mute SnapshotDisruptionIT.testDisruptionOnSnapshotInitialization + + Relates #36779 + +commit 103c4d4762f52d5a946fd8b4051ca92ba3fcc884 +Author: Tanguy Leroux +Date: Tue Dec 18 17:43:14 2018 +0100 + + [Close Index API] Mark unavailable shard copy as stale during verification (#36755) + + This pull request modifies the TransportVerifyShardBeforeCloseAction so that + it marks unavailable shards as stale. + +commit bd12e00b5305570bb6442026518de0989dcd2981 +Author: Luca Cavanna +Date: Tue Dec 18 10:33:12 2018 +0100 + + Update versions in SearchSortValues transport serialization + + Now that #36617 is backported to 6.x, the version in the transport serialization conditionals for the search sort values can be updated to 6.6.0 + +commit 1aad08c0c089535b889f42d9f746cc3890e10d1b +Author: Luca Cavanna +Date: Mon Dec 17 12:53:43 2018 +0100 + + Update version in SearchHits transport serialization + + Now that #36555 is backported to 6.x, the version in the transport serialization conditionals for sort and collapse info can be updated to 6.6.0 + +commit ec0dc2c0e900e1703dea29ddb56e4916618fc49e +Author: Nick Knize +Date: Tue Dec 18 09:54:56 2018 -0600 + + [Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#36751) + + * [Geo] Expose BKDBackedGeoShapes as new VECTOR strategy + + This commit exposes lucene's LatLonShape field as a new + strategy in GeoShapeFieldMapper. To use the new indexing + approach, strategy should be set to "vector" in the + geo_shape field mapper. If the tree parameter is set + the mapper will throw an IAE. Note the following: + + When using vector strategy: + + * geo_shape query does not support querying by POINT, + MULTIPOINT, or GEOMETRYCOLLECTION. + * LINESTRING and MULTILINESTRING queries do not support + WITHIN relation. + * CONTAINS relation is not supported. + * The tree, precision, tree_levels, distance_error_pct, + and points_only parameters will not throw an exception + but they have no effect and will be marked as + deprecated.. + + All other features are supported. + + * revert change to PercolatorFieldMapper + + * fix ExistsQuery for geo_shape vector strategy + + * add deprecation logging for tree, precision, tree_levels, distance_error_pct, and points_only + + * initial update to geoshape docs, including mapping migration updates + + * initial support for GeoCollection queries + + * fix docs and javadoc errors + + * clean up geocollection queries + + * set deprecated mapping tests to NOTCONSOLE + + * fix geo-shape mapper asciidoc mapping and test warnings + + * add support for point queries using LatLonShapeBoundingBoxQuery + + * update GeoShapeQueryBuilderTests to include POINT queries for VECTOR strategy. Other comment cleanups + + * add lucene geometry build testing to ShapeBuilder tests + + * remove deprecated prefix tree mapping from geo-shape.asciidoc + + * refactor GeoShapeFieldMapper into LegacyGeoShapeFieldMapper and GeoShapeFieldMapper + + Both classes derive from BaseGeoShapeFieldMapper that provides shared parameters: + coerce, ignoreMalformed, ignore_z_value, orientation. + + * update docs to remove vector strategy + + * fix GeometryCollectionBuilder#buildLucene to return the object created by the shape builder + + * fix LineLength failure in GeoJsonShapeParserTests + + * ShapeMapper refactor changes from PR feedback + + * fix typo in geo-shape.asciidoc + + * ignore circle test in docs + + * update indexing-approach ref to geoshape-indexing-approach + + * add warnings check for LegacyGeoShapeFieldMapper to AbstractBuilderTestCase + + * fix deprecatedParameters setup + + * update indexing approach + + * fixing unexpected warnings failures + + * move orientation back to field type + + * remove if in LegacyGeoShapeFieldMapper#doXContent. Fix GeoShapeFieldMapper to work with double array as a point + + * fix indexing-approach link in circle section of geoshape docs + + * add strategy to deprecation warnings check + + * fix test failures + + * fix typo in QueryStringQueryBuilderTests + + * fix total hits to totalHits().value + + * fix version number + + * add version check to BaseGeoShapeFieldMapper + + * fix line length! + + * revert version check in BaseGeoShapeFieldMapper + + * Fix serialization of mappings of legacy shapes. + +commit 41feaf137ce0c057cd96af7601f6285b5178793c +Author: Christoph Büscher +Date: Tue Dec 18 16:54:06 2018 +0100 + + [Docs] Fix error in Common Grams Token Filter (#36774) + + The first example given is missing the two single-token cases for "is" and "a". + The later usage example is slightly wrong in that custom analyzers should + go under `settings.analysis.analyzer`. + +commit f05c404934602b555bfc039244d8a1c1a54fea4a +Author: Jim Ferenczi +Date: Tue Dec 18 22:16:33 2018 +0700 + + Fix rollup search statistics (#36674) + + This change fixes the rollup statistics regarding search times. Search times are + computed from the first query and never updated. This commit adds the missing + calls to the subsequent search. + +commit 74502d7f4b8bdb78004729a90b5f5ae1526f282b +Author: Marios Trivyzas +Date: Tue Dec 18 17:07:27 2018 +0200 + + SQL: Fix wrong appliance of StackOverflow limit for IN (#36724) + + Fix grammar so that each element inside the list of values for IN + is a valueExpression and not a more generic expression. Introduce a + mapping for context names as some rules in the grammar are exited with + a different rule from the one they entered.This helps so that the decrement + of depth counts in the Parser's CircuitBreakerListener works correctly. + + For the list of values for IN, don't count the + PrimaryExpressionContext as this is not visited on exitRule() due to + the peculiarity in our gramamr with the predicate and predicated. + + Fixes: #36592 + +commit 1afcfc97bd5f514e17b78f393c1b8d3a44feb75b +Author: Martijn van Groningen +Date: Tue Dec 18 16:00:09 2018 +0100 + + [TEST] Added more logging + + Relates to #36761 + +commit cd3a1af77bf98c9646d41848b71252f7141ce34d +Merge: 0a0c969517f 78f9af19c66 +Author: Tanguy Leroux +Date: Tue Dec 18 15:52:39 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 78f9af19c668886c682d5d1d74f5fb6f2a4c5e77 +Author: Ioannis Kakavas +Date: Tue Dec 18 16:12:43 2018 +0200 + + Invalidate Token API enhancements - HLRC (#36362) + + * Adds Invalidate Token API enhancements to HLRC + + Relates: #35388 + +commit f884b2b1cda5a0abae6c5b2db28dec58f9108525 +Author: Mayya Sharipova +Date: Tue Dec 18 08:53:49 2018 -0500 + + Deprecate types in index API (#36575) + + * Deprecate types in index API + + - deprecate type-based constructors of IndexRequest + - update tests to use typeless IndexRequest constructors + - no yaml tests as they have been already added in #35790 + + Relates to #35190 + +commit 1a46d15bbdc8300c7a9138d2b9247715c8841f4e +Author: Luca Cavanna +Date: Tue Dec 18 14:47:34 2018 +0100 + + Disable bwc tests until #36555 backport is complete (#36737) + +commit 5f76f39386af17899bd3a495b2f501fb1066241f +Author: Boaz Leskes +Date: Tue Dec 18 14:35:18 2018 +0100 + + Rename seq# powered optimistic concurrency control parameters to ifSeqNo/ifPrimaryTerm (#36757) + + This PR renames the parameters previously introduce to the following: + + ### URL Parameters + ``` + PUT twitter/_doc/1?if_seq_no=501&if_primary_term=1 + { + "user" : "kimchy", + "post_date" : "2009-11-15T14:12:12", + "message" : "trying out Elasticsearch" + } + + DELETE twitter/_doc/1?if_seq_no=501&if_primary_term=1 + ``` + + ### Bulk API + ``` + POST _bulk + { "index" : { "_index" : "test", "_type" : "_doc", "_id" : "1", "if_seq_no": 501, "if_primary_term": 1 } } + { "field1" : "value1" } + { "delete" : { "_index" : "test", "_type" : "_doc", "_id" : "2", "if_seq_no": 501, "if_primary_term": 1 } } + ``` + + ### Java API + ``` + IndexRequest.ifSeqNo(long seqNo) + IndexRequest.ifPrimaryTerm(long primaryTerm) + DeleteRequest.ifSeqNo(long seqNo) + DeleteRequest.ifPrimaryTerm(long primaryTerm) + ``` + + Relates #36148 + Relates #10708 + +commit af57575838145d48b0f37ce9c592228a0e6a97b3 +Author: Alan Woodward +Date: Tue Dec 18 13:20:51 2018 +0000 + + Allow word_delimiter_graph_filter to not adjust internal offsets (#36699) + + This commit adds an adjust_offsets parameter to the word_delimiter_graph token filter, defaulting + to true. Most of the time you'd want sub-tokens emitted by this filter to have offsets that are + adjusted to their real position in the token stream; however, some token filters can change the + length or starting position of a token (eg trim) without changing their offset attributes, and this + can lead to word_delimiter_graph emitting illegal offsets. Setting adjust_offsets to false in these + cases will allow indexing again. + + Fixes #34741, #33710 + +commit 0ff1f1fa18401cebca19c728e1735273ea5d7c87 +Author: Martijn van Groningen +Date: Tue Dec 18 13:38:51 2018 +0100 + + Muted tests. + + Relates to #36764 + +commit 624307410e3d5fceffcc7d91752bdf61fa843f02 +Author: David Roberts +Date: Tue Dec 18 12:18:29 2018 +0000 + + [ML] Create the ML annotations index (#36731) + + The ML UI now provides the ability for users to annotate + time periods with arbitrary text to add insight to what + happened. + + This change makes the backend create the index for these + annotations, together with read and write aliases to + make future upgrades possible without adding complexity + to the UI. + + It also adds read and write permission to the index for + all ML users (not just admins). + + The spec for the index is in + https://github.com/elastic/kibana/pull/26034/files#diff-c5c6ac3dbb0e7c91b6d127aa06121b2cR7 + + Relates #33376 + Relates elastic/kibana#26034 + +commit 4bc9bffb4d50d0e8b1a8070cf57ff429fbf013fa +Author: Andrei Stefan +Date: Tue Dec 18 13:29:49 2018 +0200 + + SQL: Extend the ODBC metric by differentiating between 32 and 64bit platforms (#36753) + + * The "overall" ODBC metric will now track a total of requests between 32bit and 64bit ODBC metrics, being calculated passively, upon request. + +commit 9087c98a5aff8d55205383aa597c8aced740c94f +Author: Boaz Leskes +Date: Tue Dec 18 10:56:02 2018 +0100 + + Expose Sequence Number based Optimistic Concurrency Control in the rest layer (#36721) + + Relates #36148 + Relates #10708 + +commit 57e1a4bc9f60de59937c0d647d7b26af89b54923 +Author: Martijn van Groningen +Date: Tue Dec 18 10:43:11 2018 +0100 + + [TEST] Ensure shard follow tasks have really stopped. + + Relates to #36696 + +commit 8a412c6a266a84cfb2b7009f57d262fa1559d955 +Author: Luca Cavanna +Date: Tue Dec 18 10:43:14 2018 +0100 + + Ensure MapperService#getAllMetaFields elements order is deterministic (#36739) + + MapperService#getAllMetaFields returns an array, which is created out of + an `ObjectHashSet`. Such set does not guarantee deterministic hash + ordering. The array returned by its toArray may be sorted differently + at each run. This caused some repeatability issues in our tests (see #29080) + as we pick random fields from the array of possible metadata fields, + but that won't be repeatable if the input array is sorted differently at + every run. Once setting the tests seed, hppc picks that up and the sorting is + deterministic, but failures don't repeat with the seed that gets printed out + originally (as a seed was not originally set). + See also https://issues.carrot2.org/projects/HPPC/issues/HPPC-173. + + With this commit, we simply create a static sorted array that is used for + `getAllMetaFields`. The change is in production code but really affects + only testing as the only production usage of this method was to iterate + through all values when parsing fields in the high-level REST client code. + Anyways, this seems like a good change as returning an array would imply + that it's deterministically sorted. + +commit 00521a5b36fe2d91e9c669ff4f0ebd5a2b3f39f5 +Author: Alexander Reelsen +Date: Tue Dec 18 10:11:39 2018 +0100 + + Watcher: Ensure all internal search requests count hits (#36697) + + In previous commits only the stored toXContent version of a search + request was using the old format. However an executed search request was + already disabling hit counts. In 7.0 hit counts will stay enabled by + default to allow for proper migration. + + Closes #36177 + +commit 0a0c969517f48966a452a24da9064edc0507be92 +Merge: 7372529a953 b57e12aa44c +Author: Tanguy Leroux +Date: Tue Dec 18 09:27:35 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 7372529a95393114775c009501bd155abf310d07 +Author: Tanguy Leroux +Date: Tue Dec 18 09:25:47 2018 +0100 + + [Tests] Reduce randomization in CloseWhileRelocatingShardsIT (#36694) + +commit b57e12aa44cfe6aa8d1f708824de5933d85b4cad +Author: Luca Cavanna +Date: Tue Dec 18 09:20:51 2018 +0100 + + Add raw sort values to SearchSortValues transport serialization (#36617) + + In order for CCS alternate execution mode (see #32125) to be able to do the final reduction step on the CCS coordinating node, we need to serialize additional info in the transport layer as part of each `SearchHit`. Sort values are already present but they are formatted according to the provided `DocValueFormat` provided. The CCS node needs to be able to reconstruct the lucene `FieldDoc` to include in the `TopFieldDocs` and `CollapseTopFieldDocs` which will feed the `mergeTopDocs` method used to reduce multiple search responses (one per cluster) into one. + + This commit adds such information to the `SearchSortValues` and exposes it through a new getter method added to `SearchHit` for retrieval. This info is only serialized at transport and never printed out at REST. + +commit 7b9ca62174216097493e83d1f127c1098c4f6737 +Author: Ioannis Kakavas +Date: Tue Dec 18 10:05:50 2018 +0200 + + Enhance Invalidate Token API (#35388) + + This change: + + - Adds functionality to invalidate all (refresh+access) tokens for all users of a realm + - Adds functionality to invalidate all (refresh+access)tokens for a user in all realms + - Adds functionality to invalidate all (refresh+access) tokens for a user in a specific realm + - Changes the response format for the invalidate token API to contain information about the + number of the invalidated tokens and possible errors that were encountered. + - Updates the API Documentation + + After back-porting to 6.x, the `created` field will be removed from master as a field in the + response + + Resolves: #35115 + Relates: #34556 + +commit 96d279ed83e25f427db86b831333ca6772202a91 +Author: Nicholas Knize +Date: Mon Dec 17 20:09:46 2018 -0600 + + Revert "[Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320)" + + This reverts commit 5bc7822562a6eefa4a64743233160cdc9f431adf. + +commit 2f5300e3a626c3dbbebe5122d0e724cc1aa8f551 +Author: Christoph Büscher +Date: Tue Dec 18 00:57:42 2018 +0100 + + Deprecate types in get_source and exist_source (#36426) + + This change adds a new untyped endpoint `{index}/_source/{id}` for both the + GET and the HEAD methods to get the source of a document or check for its + existance. It also adds deprecation warnings to RestGetSourceAction that emit + a warning when the old deprecated "type" parameter is still used. Also updating + documentation and tests where appropriate. + + Relates to #35190 + +commit f0f2b261595e2f5ea48736eac7a4e4121dd20b2c +Author: Jason Tedor +Date: Mon Dec 17 17:42:58 2018 -0500 + + Fix duplicate phrase in shrink/split error message (#36734) + + This commit removes a duplicate "must be a" from the shrink/split error + messages. + +commit 384757deffb1bd88781c2d5c3459ca25f15792a3 +Author: Jake Landis +Date: Mon Dec 17 16:25:11 2018 -0600 + + ingest: support default pipelines + bulk upserts (#36618) + + This commit adds support to enable bulk upserts to use an index's + default pipeline. Bulk upsert, doc_as_upsert, and script_as_upsert + are all supported. + + However, bulk script_as_upsert has slightly surprising behavior since + the pipeline is executed _before_ the script is evaluated. This means + that the pipeline only has access the data found in the upsert field + of the script_as_upsert. The non-bulk script_as_upsert (existing behavior) + runs the pipeline _after_ the script is executed. This commit + does _not_ attempt to consolidate the bulk and non-bulk behavior for + script_as_upsert. + + This commit also adds additional testing for the non-bulk behavior, + which remains unchanged with this commit. + + fixes #36219 + +commit 75bfbe92eec7ee69e3cc3bfccaa39fbc78489e00 +Author: Armin Braun +Date: Mon Dec 17 22:34:27 2018 +0100 + + TESTS:Debug Log. IndexStatsIT#testFilterCacheStats + +commit 5bc7822562a6eefa4a64743233160cdc9f431adf +Author: Nick Knize +Date: Mon Dec 17 14:38:14 2018 -0600 + + [Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (#35320) + + This commit exposes lucene's LatLonShape field as the + default type in GeoShapeFieldMapper. To use the new + indexing approach, simply set "type" : "geo_shape" in + the mappings without setting any of the strategy, precision, + tree_levels, or distance_error_pct parameters. Note the + following when using the new indexing approach: + + * geo_shape query does not support querying by + MULTIPOINT. + * LINESTRING and MULTILINESTRING queries do not + yet support WITHIN relation. + * CONTAINS relation is not yet supported. + The tree, precision, tree_levels, distance_error_pct, + and points_only parameters are deprecated. + +commit f1e1f93943ad17c1ed48b1aec7fdb9f074ea6560 +Author: Luca Cavanna +Date: Mon Dec 17 21:19:39 2018 +0100 + + [TEST] fix float comparison in RandomObjects#getExpectedParsedValue + + This commit fixes a test bug introduced with #36597. This caused some + test failure as stored field values comparisons would not work when CBOR + xcontent type was used. + + Closes #29080 + +commit 3dd5a5a3c5ff73e7798e54d84a36ff735e72be55 +Author: Tim Brooks +Date: Mon Dec 17 13:19:32 2018 -0700 + + Initialize startup `CcrRepositories` (#36730) + + Currently, the CcrRepositoryManger only listens for settings updates + and installs new repositories. It does not install the repositories that + are in the initial settings. This commit, modifies the manager to + install the initial repositories. Additionally, it modifies the ccr + integration test to configure the remote leader node at startup, instead + of using a settings update. + +commit 7bf822bbbb7a624d80825b50e0506036bcd68691 +Author: Jake Landis +Date: Mon Dec 17 14:10:13 2018 -0600 + + ingest: fix on_failure with Drop processor (#36686) + + This commit allows a document to be dropped when a Drop processor + is used in the on_failure fork of the processor chain. + + Fixes #36151 + +commit 4d0bb9dd0a5cf1dfe1104b4aa8de9e56f5aa80f1 +Author: Armin Braun +Date: Mon Dec 17 21:02:59 2018 +0100 + + SNAPSHOTS: Adjust BwC Versions in Restore Logic (#36718) + + * Re-enables bwc tests with adjusted version conditions now that #36397 enables concurrent snapshots in 6.6+ + +commit a0e7e571e448d1c4f91e641dc411d51023f42ebc +Author: Jack Conradson +Date: Mon Dec 17 10:50:19 2018 -0800 + + [Painless] Add boxed type to boxed type casts for method/return (#36571) + + This adds implicit boxed type to boxed types casts for non-def types to create asymmetric casting relative to the def type when calling methods or returning values. This means that a user calling a method taking an Integer can call it with a Byte, Short, etc. legally which matches the way def works. This creates consistency in the casting model that did not previously exist. + +commit cd632de11636d5995d2e1c8c8cfe4abd001a9642 +Author: Jason Tedor +Date: Mon Dec 17 13:24:31 2018 -0500 + + Do not resolve addresses in remote connection info (#36671) + + The remote connection info API leads to resolving addresses of seed + nodes when invoked. This is problematic because if a hostname fails to + resolve, we would not display any remote connection info. Yet, a + hostname not resolving can happen across remote clusters, especially in + the modern world of cloud services with dynamically chaning + IPs. Instead, the remote connection info API should be providing the + configured seed nodes. This commit changes the remote connection info to + display the configured seed nodes, avoiding a hostname resolution. Note + that care was taken to preserve backwards compatibility with previous + versions that expect the remote connection info to serialize a transport + address instead of a string representing the hostname. + +commit 1f80c80b3ae1f99f77ea2d3a5fd519c722485fcf +Author: Andrei Stefan +Date: Mon Dec 17 19:06:23 2018 +0200 + + Add back one line removed by mistake regarding java version check and + COMPAT jvm parameter existence + +commit 27ad733dce885914f46c24b40f35d51e447fa78e +Author: Evgenia Badyanova +Date: Mon Dec 17 11:59:45 2018 -0500 + + Fixing line length for EnvironmentTests and RecoveryTests (#36657) + + Relates #34884 + +commit 6d9d5e397b9f2b008a6023dd2d65aa071efbdba2 +Author: Costin Leau +Date: Mon Dec 17 18:58:06 2018 +0200 + + SQL: Fix translation of LIKE/RLIKE keywords (#36672) + + * SQL: Fix translation of LIKE/RLIKE keywords + + Refactor Like/RLike functions to simplify internals and improve query + translation when chained or within a script context. + + Fix #36039 + Fix #36584 + +commit 4103d3b9ec5186aa74c987f72d65c99ee27f4c2b +Author: Lisa Cawley +Date: Mon Dec 17 08:18:11 2018 -0800 + + [DOCS] Adds monitoring requirement for ingest node (#36665) + +commit eb59c1f7bdc664c590da392518633846cf00075d +Author: Armin Braun +Date: Mon Dec 17 16:06:56 2018 +0100 + + SNAPSHOTS: Disable BwC Tests Until #36659 Landed (#36709) + +commit e356b8cb958cba03050fa698865c465b3bf77267 +Author: Boaz Leskes +Date: Mon Dec 17 15:22:13 2018 +0100 + + Add doc's sequence number + primary term to GetResult and use it for updates (#36680) + + This commit adds the last sequence number and primary term of the last operation that have + modified a document to `GetResult` and uses it to power the Update API. + + Relates #36148 + Relates #10708 + +commit a181a25226ce2c9de532934762c5f48d76298c69 +Author: Martijn van Groningen +Date: Mon Dec 17 14:14:56 2018 +0100 + + [CCR] Add time since last auto follow fetch to auto follow stats (#36542) + + For each remote cluster the auto follow coordinator, starts an auto + follower that checks the remote cluster state and determines whether an + index needs to be auto followed. The time since last auto follow is + reported per remote cluster and gives insight whether the auto follow + process is alive. + + Relates to #33007 + Originates from #35895 + +commit 6f038997e14b9427399df77f8ff6415fa8cfb67a +Author: Albert Zaharovits +Date: Mon Dec 17 14:45:55 2018 +0200 + + Watcher accounts constructed lazily (#36656) + + This fixes two bugs about watcher notifications: + * registering accounts that had only secure settings was not possible before; + these accounts are very much practical for Slack and PagerDuty integrations. + * removes the limitation that, for an account with both secure and cluster settings, + the admin had to first change/add the secure settings and only then add the + dependent dynamic cluster settings. The reverse order would trigger a + SettingsException for an incomplete account. + + The workaround is to lazily instantiate account objects, hoping that when accounts + are instantiated all the required settings are in place. Previously, the approach + was to greedily validate all the account settings by constructing the account objects, + even if they would not ever be used by actions. This made sense in a world where + all the settings were set by a single API. But given that accounts have dependent + settings (that must be used together) that have to be changed using different APIs + (POST _nodes/reload_secure_settings and PUT _cluster/settings), the settings group + would technically be in an invalid state in between the calls. + This fix builds account objects, and validates the settings, when they are + needed by actions. + +commit 2ed6ab9648cb6d93d6c27fe9b0db3e5a5c57e91e +Author: Andrei Stefan +Date: Mon Dec 17 14:00:56 2018 +0200 + + SQL: Concat should be always not nullable (#36601) + +commit b376edf26982f7b7756102664086448cba93e176 +Author: Costin Leau +Date: Mon Dec 17 13:55:44 2018 +0200 + + SQL: Move internals from TimeZone to ZoneId (#36651) + + As the internals have moved to java.time, the usage of TimeZone itself + should be minimized as it creates issues when being converted to ZoneId + Protocol wise the two are mostly identical so consumer should not see + any difference. + Note that terminology wise, inside the docs, the public API and inside + the protocol timeZone will continue to be used as it's more widely + understood as oppose to zoneId which is an implementation detail + specific to the JVM + + Fix #36535 + +commit 3f128a89bcac93c17b6b8edc59892d3f9404cddd +Author: Costin Leau +Date: Mon Dec 17 13:40:20 2018 +0200 + + SQL: Disable integration test due to TZ issues + + Due to the difference in TZ between H2 and SQL, disable minute test for + NOW() + + Fix #36695 + +commit b15f27f6a6bed3c349169a7c928e6c5460548bdb +Author: Andrei Stefan +Date: Mon Dec 17 13:36:48 2018 +0200 + + SQL: Scripting support for casting functions CAST and CONVERT (#36640) + +commit ef6ae699cf1f626c4f0966cb3618daef8b029136 +Author: Tanguy Leroux +Date: Mon Dec 17 10:28:26 2018 +0100 + + [Close Index API] Adapt MetaDataIndexStateServiceTests after merge + +commit 79999d37d48d254f6e521822ab455f88a857ecf8 +Merge: 1f6f236d44e c0866393ee5 +Author: Tanguy Leroux +Date: Mon Dec 17 10:14:38 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit c0866393ee5a07e6cde756af4cfdd7172ec117a2 +Author: Marios Trivyzas +Date: Mon Dec 17 11:05:52 2018 +0200 + + SQL: Fix issue with complex HAVING and GROUP BY ordinal (#36594) + + When trying to analyse a HAVING condition, we crate a temp Aggregate + Plan which wasn't created correctly (missing the expressions from the + SELECT clause) and as a result the ordinal (1, 2, etc) in the GROUP BY + couldn't be resolved correctly. + + Also after successfully analysing the HAVING condition, still the + original plan was returned. + + Fixes: #36059 + +commit f7567b865eec3f973c6bbc4a7fe3786155c15ce2 +Author: David Turner +Date: Mon Dec 17 08:34:34 2018 +0000 + + Improve assertions in testStaleMasterNotHijackingMajority (#36658) + + Investigating #36556 was made a little trickier because the feedback from the + failing assertion wasn't very informative, and the messages attached to other + nearby assertions were misleading. This commit improves the feedback from these + assertions and tidies up a few other issues in the test suite. + +commit e82a647740ae7cbd0529f5a7bf5bd2087a77d834 +Author: Armin Braun +Date: Mon Dec 17 09:20:21 2018 +0100 + + TEST: Fix Assertion for Shard Allocation Counts (#36685) + + * If removing half the nodes completely removes a shard from the cluster we can't count it in the assertion + * Also: + * Remove unused logger parameter + * Fix typo in var name + * Closes #35365 + +commit f27d2c292797c28ba73f27b5d32928d2c4a5ac2f +Author: Martijn van Groningen +Date: Mon Dec 17 07:55:27 2018 +0100 + + [TEST] Pause index following at end of test, + so that no unexpected failures happen at test teardown. + +commit 2028c2af14ff8b0ecb5ebbed29604184f74d7645 +Author: Nhat Nguyen +Date: Sun Dec 16 16:45:31 2018 -0500 + + TEST: Do not assert max_seq_of_updates if promotion + + If a primary promotion happens in the test testAddRemoveShardOnLeader, the + max_seq_no_of_updates_or_deletes on a new primary might be higher than the + max_seq_no_of_updates_or_deletes on the replicas or copies of the follower. + + Relates #36607 + +commit 97107e99e8e30c0ba14e6903b5da5c3630a44e32 +Author: Martijn van Groningen +Date: Sun Dec 16 13:57:51 2018 +0100 + + Moved test to its rightful place. + +commit 733a6d34c1d08f3a1c50fc5a283733cc994b4343 +Author: Boaz Leskes +Date: Sat Dec 15 17:59:57 2018 +0100 + + Add seq no powered optimistic locking support to the index and delete transport actions (#36619) + + This commit add support for using sequence numbers to power [optimistic concurrency control](http://en.wikipedia.org/wiki/Optimistic_concurrency_control) + in the delete and index transport actions and requests. A follow up will come with adding sequence + numbers to the update and get results. + + Relates #36148 + Relates #10708 + +commit a30e8c2fa3a8ee381c4fa6eae550681fee764cf5 (tag: v7.0.0-alpha2) +Author: Albert Zaharovits +Date: Sat Dec 15 08:34:54 2018 +0200 + + HasPrivilegesResponse use TreeSet for fields (#36329) + + For class fields of type collection whose order is not important + and for which duplicates are not permitted we declare them as `Set`s. + Usually the definition is a `HashSet` but in this case `TreeSet` is used + instead to aid testing. + +commit 4e4022b7ef3686dd02efa60d5ea940a11b6dbe33 +Author: Tim Brooks +Date: Fri Dec 14 18:06:48 2018 -0700 + + Register the TCP_CONNECT_TIMEOUT setting (#36663) + + This commit registers the TCP_CONNECT_TIMEOUT setting in the + ClusterSettings. + +commit 2dd56cf945873415f2bfd51555ffad6d29ce8e21 +Author: David Roberts +Date: Fri Dec 14 22:28:12 2018 +0000 + + [TEST] Make filestructurefinder.TimeoutCheckerTests more robust + +commit 306530043497d8c1092f7bdaeda6f8e46de601c4 +Author: Tim Brooks +Date: Fri Dec 14 14:41:04 2018 -0700 + + Unify transport settings naming (#36623) + + This commit updates our transport settings for 7.0. It generally takes a + few approaches. First, for normal transport settings, it usestransport. + instead of transport.tcp. Second, it uses transport.tcp, http.tcp, + or network.tcp for all settings that are proxies for OS level socket + settings. Third, it marks the network.tcp.connect_timeout setting for + removal. Network service level settings are only settings that apply to + both the http and transport modules. There is no connect timeout in + http. Fourth, it moves all the transport settings to a single class + TransportSettings similar to the HttpTransportSettings class. + + This commit does not actually remove any settings. It just adds the new + renamed settings and adds todos for settings that will be deprecated. + +commit dae422fb2b31f9da95a790111c0f0c4698738fc8 +Author: Michael Basnight +Date: Fri Dec 14 15:38:51 2018 -0600 + + Update joda compat methods to use compat class (#36654) + + The existing joda compat methods isEquals isAfter and isBefore all took + in a ZonedDateTime, but since all of the scripting is now using the new + JodaCompatZonedDateTime, these are changed to take that in instead. + +commit f64e872fa7d6c45faf03fea13c9e02cde7062a25 +Author: Yannick Welsch +Date: Fri Dec 14 22:29:43 2018 +0100 + + Use new cluster.initial_master_nodes setting in yml config + + Extracted from #34714 which will only be merged later. + +commit 276726aea2ff79a2264ceb9170b8b72c20bd8066 +Author: João Barbosa +Date: Fri Dec 14 21:22:54 2018 +0000 + + Added keyed response to pipeline percentile aggregations 22302 (#36392) + + Closes #22302 + +commit c7ac9ef82618096a11d64734b44a1a9821831cd1 +Author: Alan Woodward +Date: Fri Dec 14 20:30:07 2018 +0000 + + Upgrade to lucene snapshot 774e9aefbc (#36637) + + Includes LUCENE-8607: improvement to MatchAllDocsQuery + +commit 06dfd4aadc628b85cd36aa16f3b9129f470a6eba +Author: Tal Levy +Date: Fri Dec 14 11:36:18 2018 -0800 + + [TEST] fix flaky ILM tests (#36612) + + * WaitForRolloverReadyStepTests#mutateInstance sometimes did not mutate the instance + correctly + * 40_explain_lifecycle#"Test new phase still has phase_time" is not really a necessary + integration test. In addition to this, it is flaky due to the asynchronous nature of + ILM metadata population + +commit ccd1beb9b33acc9fc1990e5c9d9ea07dee62d133 +Author: Julie Tibshirani +Date: Fri Dec 14 10:47:27 2018 -0800 + + Deprecate types in update requests. (#36181) + + The following updates were made: + * Add deprecation warnings to `RestUpdateAction`, plus a test in `RestUpdateActionTests`. + * Deprecate relevant methods on the Java HLRC requests/ responses. + * Add HLRC integration tests for the typed APIs. + * Update documentation (for both the REST API and Java HLRC). + * Fix failing integration tests. + + Because of an earlier PR, the REST yml tests were already updated (one version without types, and another legacy version that retains types). + +commit 68a674ef1fcb120aa77ffdc59d76fa403664430e +Author: Martijn van Groningen +Date: Fri Dec 14 19:39:30 2018 +0100 + + [CCR] Fix follow stats API's follower index filtering feature (#36647) + + Currently always all follow stats for all follower indices are being + returned even if follow stats for only specific indices are requested. + +commit fbf88b2ab7eacb1dd8cc47750dad417dd3264d52 +Author: Tim Brooks +Date: Fri Dec 14 10:59:07 2018 -0700 + + Remove the `MockTcpTransport` (#36628) + + This commit removes all remaining usages of the `MockTcpTransport`. + Additionally it removes the `MockTcpTransport` and its test case. + +commit bb3ae18da5131ceef467a407879c9121ba1e22be +Author: Luca Cavanna +Date: Fri Dec 14 18:57:37 2018 +0100 + + Increase coverage in SearchSortValuesTests (#36597) + + SearchSortValuesTests extends now `AbstractSerializingTestCase` which removes some code duplication and standardizes the way we test `fromXContent`, serialization and equals/hashcode. + + Also, we were never creating `SearchSortValues` through their public constructor that accept an array of `DocValueFormat` together with the array of raw sort values. That is covered now, which involved some conversion from `BytesRef` to String in the test. + + Also, the previous test was not using doing any equality check against the original and parsed versions in `testFromXContent` due to values being parsed with different types in some cases, which is now covered by converting those values using a new method added to `RandomObjects`. The code was already there as part of `randomStoredFieldValues`, but it is now exposed to be used in other scenarios. + +commit 8f04536a35976c88e1446c5188c17d9763ce37c2 +Author: Luca Cavanna +Date: Fri Dec 14 18:30:29 2018 +0100 + + Add copy constructor to SearchRequest (#36641) + + For cross cluster search alternate execution mode (see #32125), we will need to take a search request that spans across multiple clusters (based on index prefixes e.g. cluster1:index, cluster2:index etc.) and split it into multiple search requests to be sent to each cluster. A copy constructor added to `SearchRequest` would make that easy and well maintainable in the future. + + Something along the same lines already happens in `BulkByScrollParallelizationHelper`, but the corresponding code went outdated as some new fields were added to `SearchRequest` which were not added to the bulk by scroll code. A copy constructor helps making the task of copying a search request maintainable over time. + +commit 973f1e7c8fa0434c23875cca795c5f0f07e97466 +Author: Luca Cavanna +Date: Fri Dec 14 18:28:45 2018 +0100 + + Remove NoopRequestBuilders (#36639) + + These classes were introduced when Action required a RequestBuilder type. + That is no longer needed, hence we can remove `NoopBulkRequestBuilder` + and `NoopSearchRequestBuilder`. + +commit b5725ed48271a85fe96d4974a0c62065b8fec74e +Author: Alpar Torok +Date: Fri Dec 14 18:31:52 2018 +0200 + + Move inputs and outputs before action (#36539) + +commit 6ee6bb55e2756e450d20f6a393896795aaecf728 +Author: Costin Leau +Date: Fri Dec 14 18:20:37 2018 +0200 + + SQL: Introduce HISTOGRAM grouping function (#36510) + + Introduce Histogram grouping function for bucketing/grouping data based + on a given range. Both date and numeric histograms are supported using + the appropriate range declaration (numbers vs intervals). + + SELECT HISTOGRAM(number, 50) AS h FROM index GROUP BY h + SELECT HISTOGRAM(date, INTERVAL 1 YEAR) AS h FROM index GROUP BY h + + In addition add multiply operator for Intervals + Add docs for intervals and histogram + + Fix #36509 + +commit 09bf93dc2ab0099186ac28e2d4eba8a7dd73134e +Author: Alan Woodward +Date: Fri Dec 14 15:14:00 2018 +0000 + + Add intervals query (#36135) + + * Add IntervalQueryBuilder with support for match and combine intervals + + * Add relative intervals + + * feedback + + * YAML test - broekn + + * yaml test; begin to add block source + + * Add block; make disjunction its own source + + * WIP + + * Extract IntervalBuilder and add tests for it + + * Fix eq/hashcode in Disjunction + + * New yaml test + + * checkstyle + + * license headers + + * test fix + + * YAML format + + * YAML formatting again + + * yaml tests; javadoc + + * Add OR test -> requires fix from LUCENE-8586 + + * Add docs + + * Re-do API + + * Clint's API + + * Delete bash script + + * doc fixes + + * imports + + * docs + + * test fix + + * feedback + + * comma + + * docs fixes + + * Tidy up doc references to old rule + +commit 278cc4c3de9378777a018400806f622355357e4d +Author: David Turner +Date: Fri Dec 14 14:16:26 2018 +0000 + + [Zen2] Improve assertion when creating fake node ID (#36544) + + Today we assert that the fake node ID is greater than the real node's ID. In + fact we want to assert that it's greater than _all_ proper UUIDs. This adds + assertions to that effect. + +commit f502ca53d4ff1b330697b01b6d9d5dec063f8581 +Author: Costin Leau +Date: Fri Dec 14 15:42:36 2018 +0200 + + SQL: Introduce NOW/CURRENT_TIMESTAMP function (#36562) + + Add CURRENT_TIMESTAMP as keyword as well function alongside NOW() + These return the current date/time for the given query, computed when + the statement reaches the server. For completeness, CURRENT_TIMESTAMP + also accepts precision as an optional parameter. + + Fix #36534 + +commit f1f3b28f5c1db4bb31aff86be69527f058f3d99e +Author: Jeff Hajewski +Date: Fri Dec 14 06:56:47 2018 -0600 + + Delete deprecated getValues from ScriptDocValues (#36183) + + * Adds deprecation logging to ScriptDocValues#getValues. + + First commit addressing issue #22919. + + `ScriptDocValues#getValues` was added for backwards compatibility but no + longer needed. Scripts using the syntax `doc['foo'].values` when + `doc['foo']` is a list should be using `doc['foo']` instead. + + * Fixes two build errors in #34279 + + * Removes unused import in ScriptDocValuesDatesTest + * Removes used of `.values` in example in diversified-sampler-aggregation.asciidoc + + * Removes use of .values from painless test. + + Part of #34279 + + * Updates tests to use `doc[foo]` syntax rather than `doc[foo].values`. + + * Removes use of `getValues()` and replaces use of `doc[foo].values` with `doc[foo]`. + + * Indentation fix. + + * Remove unnecessary list construction at previous `getValues()` callsite in ScriptDocValues.GeoPoints. + + * Update migration doc and add link to `getValue` in ScriptDocValues javadoc. + + * Fix compile + + * Fix javadoc issue + + * Removes ScriptDocValues#getValues usage from painless whitelist. + +commit db8f07c665d4cf4f10f0ff221572c8ebda6e50b7 +Author: Marios Trivyzas +Date: Fri Dec 14 14:51:12 2018 +0200 + + SQL: Fix translation to painless for conditionals (#36636) + + Add missing `formatTemplate()` for conditional functions which + resulted in incomplete painless script. Moreover the specific + return type of Object in the painless signatures resulted in + casting exceptions when conditional functions are used in the + ORDER BY. + + Fixes: #36631 + +commit 5df93218d58c521ffe61de6b5506b6ef58299d41 +Author: Armin Braun +Date: Fri Dec 14 13:24:29 2018 +0100 + + SNAPSHOTS: Upgrade GCS Dependencies to 1.55.0 (#36634) + + * Closes #35459 + * Closes #35229 + +commit 245cdd8c341799954315bfd846a49ec2b25a95dc +Author: David Turner +Date: Fri Dec 14 12:16:09 2018 +0000 + + Register ResyncTask.Status as a NamedWriteable (#36610) + + Today, ResyncTask.Status is not registered, but appears as a task status + sometimes, leading to `Failed to deserialize response from handler` exceptions: + + java.lang.IllegalArgumentException: Unknown NamedWriteable [org.elasticsearch.tasks.Task$Status][resync] + + This commit adds the missing registration. + +commit 44ba9ab04d8d86e754d97e5b07d39ace07eb0449 +Author: David Turner +Date: Fri Dec 14 12:15:42 2018 +0000 + + Fix racy use of ConcurrentHashMap (#36603) + + ConcurrentHashMap does not always behave correctly if removing elements and + concurrently checking for its emptyiness. Work around this by protecting all + usages with a mutex (there was only one usage unprotected by the mutex anyway) + and then we don't even need a ConcurrentHashMap at all. + +commit 7dc3d3b78b97c534280c38cff07bf8fd10a83f9f +Author: Luca Cavanna +Date: Fri Dec 14 12:22:54 2018 +0100 + + Add sort and collapse info to SearchHits transport serialization (#36555) + + In order for CCS alternate execution mode (see #32125) to be able to do the final reduction step on the CCS coordinating node, we need to serialize additional info in the transport layer as part of the `SearchHits`, specifically: + + - lucene `SortField[]` which contains info about the fields that sorting was performed on and their type, which depends on mappings (that the CCS node does not know about) + - collapse field (`String`) that field collapsing was executed on, if requested + - collapse values (`Object[]`) that field collapsing was based on, if requested + + This info is needed to be able to reconstruct the `TopFieldDocs` or `CollapseFieldTopDocs` in the CCS coordinating node to feed the `mergeTopDocs` method and reduce multiple search responses received (one per cluster) into one. + + This commit adds such information to the `SearchHits` class. It's nullable info that is not serialized through the REST layer. `SearchPhaseController` sets such info at the end of the hits reduction phase. + +commit c5b3ac557857da9d8571e4fdcb8eaf6d4a7b7b11 +Author: Armin Braun +Date: Fri Dec 14 11:39:23 2018 +0100 + + SNAPSHOTS: Allow Parallel Restore Operations (#36397) + + * Enable parallel restore operations + * Add uuid to restore in progress entries to uniquely identify them + * Adjust restore in progress entries to be a map in cluster state + * Added tests for: + * Parallel restore from two different snapshots + * Parallel restore from a single snapshot to different indices to test uuid identifiers are correctly used by `RestoreService` and routing allocator + * Parallel restore with waiting for completion to test transport actions correctly use uuid identifiers + +commit 1ae53e6b20771fb29dec0095dc3a996e28540c08 +Author: Nhat Nguyen +Date: Fri Dec 14 05:25:36 2018 -0500 + + TEST: Disable merges in testRebuildLocalCheckpointTracker (#36625) + + In this test, we keep track of a list of index commits then verify that + we reload exactly every operation from the safe commit. If a background + merge is triggered, then we might have a new index commit which is not + recorded in the tracking list. This change disables merges in the test. + + Closes #36470 + +commit 573b6325e7cd072cc37258519a9b12fc93fd7a8c +Author: Christoph Büscher +Date: Fri Dec 14 11:10:45 2018 +0100 + + [Tests] Fix edge case in SimpleQueryStringBuilderTests (#36611) + + A previous fix of a similar problem in #35201 wasn't general enough, we also + need to catch cases where the randomly generated query string starts with some + version of "now" and hits a date field. + + Closes #36595 + +commit 5fe4c9076c187224a0f93830e66f1cc997ddded1 +Author: David Pilato +Date: Fri Dec 14 10:30:39 2018 +0100 + + Default ec2 endpoint is ec2.us-east-1.amazonaws.com (#30323) + + Today our docs claim that the `discovery-ec2` plugin auto-detects the EC2 + endpoint, but this is not the case. This fixes these docs. + + Closes #27464. + +commit b7ba2fa74b10486b88bdbdd6ddf91f8091945f85 +Author: Daniel Mitterdorfer +Date: Fri Dec 14 10:07:27 2018 +0100 + + Fix line length for VersionTests + + Relates #34884 + Relates #36585 + +commit 42f094952b43487060158492ec6d3c4c7e037de1 +Author: Daniel Mitterdorfer +Date: Fri Dec 14 10:00:37 2018 +0100 + + Fix line length for IndexLifecycleActionIT + + Relates #34884 + Relates #36586 + +commit 8a857983f7c0039b0782ee4fceef96895fa96cda +Author: Tal Levy +Date: Fri Dec 14 00:24:56 2018 -0800 + + [TEST] fix SecurityDocumentationIT#testGetUsers (#36622) + + this test was failing because of two reason + - it was creating invalid users with passwords shorter than 6 characters + - it was expecting 7 total users to be returned, but it should be 9 + +commit 690b10a4a1ad402433f12837ff9cdd31872e95f8 +Author: David Roberts +Date: Fri Dec 14 07:18:09 2018 +0000 + + [ML] Interrupt Grok in file structure finder timeout (#36588) + + The file structure finder has timeout functionality, + but prior to this change it would not interrupt a + single long-running Grok match attempt. + + This commit hooks into the ThreadWatchdog facility + provided by the Grok library to interrupt individual + Grok matches that may be running at the time the + file structure finder timeout expires. + +commit a4b32f1143392beae867a88aa56f8b3d15862dd0 (es/feature-search-as-you-type) +Author: Nhat Nguyen +Date: Thu Dec 13 19:02:13 2018 -0500 + + Remove concurrency in testFailLeaderReplicaShard (#36607) + + testFailLeaderReplicaShard periodically fails because we concurrently + index to the leader group and close one of its replicas. If a + replication request hits a closing shard, we will fail that shard; + however, failing a shard is supported by the test framework - this makes + the test fail. + +commit 0804c27feeffb32b92d724264a7c19fe635881da +Author: Nicholas Knize +Date: Thu Dec 13 17:34:53 2018 -0600 + + fix error with SecurityDocumentationIT#testGetUsers + +commit 730b154c932bad5dd239069a24aa0ff3357cc0d0 +Author: Marios Trivyzas +Date: Fri Dec 14 00:59:42 2018 +0200 + + SQL: Fix MOD() for long and integer arguments (#36599) + + Previously, Math.floorMod was used for integers and longs + which has different logic for negative numbers. Also, the + priority of data types check was wrong as if one of the args + is double the evaluation should be with doubles, then for floats, + then longs and finally integers. + + Fixes: #36364 + +commit 1270d85ae19b9405b3949ab17d4a50b02c96ea77 +Author: Andrei Stefan +Date: Thu Dec 13 23:29:32 2018 +0200 + + SQL: be lenient for tests involving comparison to H2 but strict for csv spec tests (#36498) + + * Add a lenient parameter for floating point numbers and mark csv-spec + tests as not lenient vs. sql-spec tests to be lenient. + +commit 692cff830a43d7207fac02fe143cd357b08b6a31 +Author: Jason Tedor +Date: Thu Dec 13 16:14:59 2018 -0500 + + Add guidance on using CCR with Logstash (#36609) + + * Add guidance on using CCR with Logstash + + This commit adds a note to the documentation regarding how to configure + Logstash indices in the context of being available as leader indices for + cross-cluster replication. + + * Oh okay + + * idk + + * notconsole + +commit 42d76a7e86596fac92915b45fa010984d137d513 +Author: Chris Earle +Date: Thu Dec 13 16:13:26 2018 -0500 + + [Monitoring] LocalExporter should catch synchronous exceptions (#36606) + + In the unlikely scenario that `LocalExporter::resolveBulk` throws + an exception, then we should mark the `listener` as having failed. + +commit 64da98070c36425273fc6008521e28f8e26a9596 +Author: Gil Raphaelli +Date: Thu Dec 13 15:44:34 2018 -0500 + + format numbered list (#36580) + +commit 254d1e8f22d49f836cfdaa77442384749965f4ea +Author: Ryan Ernst +Date: Thu Dec 13 12:26:51 2018 -0800 + + Core: Add backcompat for joda time formats (#36531) + + This commit adds deprecation warnings when using format specifiers with + joda data formats that will change with java time. It also adds the "8" + prefix which may be used to force the new java time format parsing. + +commit b33ff16d62e57843c407271d3aadeb6328567441 +Author: Christoph Büscher +Date: Thu Dec 13 20:13:16 2018 +0100 + + Remove deprecated `useDisMax` from MultiMatchQuery (#36488) + + The getters and setters for useDisMax() have been deprecated since at least 6.0, + also there hasn't been any reference to the query parameter in the + documentation. Removing it from the builder and tests and replacing it with + `tieBreaker(1.0f)` where necessary. + +commit 4b17055035daee9f6f569112fdffd9c93d4c650f +Author: Nick Knize +Date: Thu Dec 13 12:24:48 2018 -0600 + + HLRC: Add get users action (#36332) + + This commit adds get user action to the high level rest client. + +commit 1f6f236d44e0d0547a36b637875d1af91d10802d +Merge: 8e5dd20efb6 b820d7c617d +Author: Tanguy Leroux +Date: Thu Dec 13 17:40:06 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 8e5dd20efb667d272fac4b2a151f8c09f0284024 +Author: Tanguy Leroux +Date: Thu Dec 13 17:36:23 2018 +0100 + + [Close Index API] Refactor MetaDataIndexStateService (#36354) + + The commit changes how indices are closed in the MetaDataIndexStateService. + It now uses a 3 steps process where writes are blocked on indices to be closed, + then some verifications are done on shards using the TransportVerifyShardBeforeCloseAction + added in #36249, and finally indices states are moved to CLOSE and their routing + tables removed. + + The closing process also takes care of using the pre-7.0 way to close indices if the + cluster contains mixed version of nodes and a node does not support the TransportVerifyShardBeforeCloseAction. It also closes unassigned indices. + + Related to #33888 + +commit b820d7c617de8dd75e36d6e58cdda9026e408462 +Author: Tal Levy +Date: Thu Dec 13 08:17:36 2018 -0800 + + fix MultiValuesSourceFieldConfig toXContent (#36525) + + This commit turns MultiValuesSourceFieldConfig into a proper + ToXContentObject for easy testing and verification of its + to/from XContent methods. + + Closes #36474. + +commit e3cf64229992cbc9318c8e96c2534826bd916045 +Author: Tal Levy +Date: Thu Dec 13 08:11:33 2018 -0800 + + Add ILM-specific security privileges (#36493) + + * add read_ilm cluster privilege + + Although managing ILM policies is best done using the + "manage" cluster privilege, it is useful to have read-only + views. + + * adds `read_ilm` cluster privilege for viewing policies and status + * adds Explain API to the `view_index_metadata` index privilege + + * add manage_ilm privileges + +commit f998e04c345760583740ede7b3d2b15ee777619f +Author: Tim Brooks +Date: Thu Dec 13 09:10:18 2018 -0700 + + Remove usages of `MockTcpTransport` from zen tests (#36579) + + We are attempting to replace the usage of the `MockTcpTransport` with + the `MockNioTransport`. This commit replaces usages of + `MockTcpTransport` in two zen test cases. + +commit 2afa7faefd0cec893ad83298e43f6342d02c2d85 +Author: Jason Tedor +Date: Thu Dec 13 10:23:45 2018 -0500 + + Override the JVM DNS cache policy (#36570) + + When a security manager is present, the JVM will cache positive hostname + lookups indefinitely. This can be problematic, especially in the modern + world with cloud services where DNS addresses can change, or + environments using Docker containers where IP addresses could be + considered ephemeral. This behavior impacts cluster discovery, + cross-cluster replication and cross-cluster search, reindex from remote, + snapshot repositories, webhooks in Watcher, external authentication + mechanisms, and the Elastic Stack Monitoring Service. The experience of + watching a DNS lookup change yet not be reflected within Elasticsearch + is a poor experience for users. The reason the JVM has this is guard + against DNS cache posioning attacks. Yet, there is already a defense in + the modern world against such attacks: TLS. With proper certificate + validation, even if a resolver falls prey to a DNS cache poisoning + attack, using TLS would neuter the attack. Therefore we have a policy + with dubious security value that significantly impacts usability. As + such we make the usability/security tradeoff towards usability, since + the security risks are very low. This commit introduces new system + properties that Elasticsearch observes to override the JVM DNS cache + policy. + +commit 40953d70caf292dd0a3d105a8c0dd1a83ba347c0 +Merge: dbcafbefc9c 7446f753465 +Author: Tanguy Leroux +Date: Thu Dec 13 15:45:56 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 7446f75346576dc816a8147699f1f43122839423 +Author: Igor Motov +Date: Thu Dec 13 17:32:45 2018 +0400 + + Geo: Adds a name of the field to geopoint parsing errors (#36529) + + Adds the field name and type to geo_point parsing errors. + + Closes #15965 + +commit 4d739d71d738ac4815a31137ac3d4ebb1b3a3b96 +Author: Daniel Mitterdorfer +Date: Thu Dec 13 14:09:50 2018 +0100 + + Document dynamic email settings in Watcher + + With this commit we document which of the email notification settings in + Watcher can be set dynamically via the cluster settings API. + + Relates #36543 + +commit bda03163e70cf9eebff739be83adad0937747736 +Author: Mayya Sharipova +Date: Thu Dec 13 07:17:52 2018 -0500 + + Make vector fields experimental feature + + Relates to #33022 + +commit 347468e916e000856a0e377ca14c17054364ffbc +Author: Andrei Stefan +Date: Thu Dec 13 14:05:16 2018 +0200 + + SQL: do not ignore all fields whose names start with underscore (#36214) + + * Do not ignore fields whose names start with underscore, unless they are + meta fields. + * Filter out _size field. + +commit 088d3f39c305ca3746ab27cc631cf08f84795b21 +Author: Armin Braun +Date: Thu Dec 13 12:54:22 2018 +0100 + + MINOR: Cleanup Dead Code in o.e.c.metadata Package (#36587) + +commit 13cb0fb98b12ca3e058382186f8175ab1db84717 +Author: David Roberts +Date: Thu Dec 13 09:15:27 2018 +0000 + + Periodically try to reassign unassigned persistent tasks (#36069) + + Previously persistent task assignment was checked in the + following situations: + + - Persistent tasks are changed + - A node joins or leaves the cluster + - The routing table is changed + - Custom metadata in the cluster state is changed + - A new master node is elected + + However, there could be situations when a persistent + task that could not be assigned to a node could become + assignable due to some other change, such as memory + usage on the nodes. + + This change adds a timed recheck of persistent task + assignment to account for such situations. The timer + is suspended while checks triggered by cluster state + changes are in-flight to avoid adding burden to an + already busy cluster. + + Closes #35792 + +commit 34d7cc13c1e9337348ca05b828202a6ccd3fbb3e +Author: Armin Braun +Date: Thu Dec 13 10:02:54 2018 +0100 + + SETTINGS: Correctly Identify Noop Updates (#36560) + + * We should compare the target value with the to be applied value before interpreting the update as a change + * This speeds up the test failing in #36496 considerably by preventing state updates on noop setting updates + +commit c650be72cfc311c29ac4f3743a42491e2571b998 +Author: Tim Vernum +Date: Thu Dec 13 19:39:59 2018 +1100 + + [TEST] Prevent duplicate key in XContent test (#36581) + + It was possible to generate a response that would output duplicate + keys + + Closes: #36473 + +commit f6b5d7e01387c09a449f67e2678cf852f58770b1 +Author: Boaz Leskes +Date: Thu Dec 13 08:08:40 2018 +0100 + + Add sequence numbers based optimistic concurrency control support to Engine (#36467) + + This commit add support to engine operations for resolving and verifying the sequence number and + primary term of the last modification to a document before performing an operation. This is + infrastructure to move our (optimistic concurrency control)[http://en.wikipedia.org/wiki/Optimistic_concurrency_control] API to use sequence numbers instead of internal versioning. + + Relates #36148 + Relates #10708 + +commit cd1bec3a06df97a5ec7c135f475d0059dc1bee64 +Author: Tal Levy +Date: Wed Dec 12 21:07:21 2018 -0800 + + [refactor] add Environment in BootstrapContext (#36573) + + There are certain BootstrapCheck checks that may need access environment-specific + values. Watcher's EncryptSensitiveDataBootstrapCheck passes in the node's environment + via a constructor to bypass the shortcoming in BootstrapContext. This commit + pulls in the node's environment into BootstrapContext. + + Another case is found in #36519, where it is useful to check the state of the + data-path. Since PathUtils.get and Paths.get are forbidden APIs, we rely on + the environment to retrieve references to things like node data paths. + + This means that the BootstrapContext will have the same Settings used in the + Environment, which currently differs from the Node's settings. + +commit d40037c91e9487f1b8ccd7f2957c4b489b8f43cf +Author: Mayya Sharipova +Date: Wed Dec 12 21:21:53 2018 -0500 + + Deprecate uses of _type as a field name in queries (#36503) + +commit b5d532f9e3d184d4bb895835a4d4fef2fb4ee0e8 +Author: Mayya Sharipova +Date: Wed Dec 12 21:20:53 2018 -0500 + + Vector field (#33022) + + 1. Dense vector + + PUT dindex + { + "mappings": { + "_doc": { + "properties": { + "my_vector": { + "type": "dense_vector" + }, + "my_text" : { + "type" : "keyword" + } + } + } + } + } + + PUT dinex/_doc/1 + { + "my_text" : "text1", + "my_vector" : [ 0.5, 10, 6 ] + } + + 2. Sparse vector + + PUT sindex + { + "mappings": { + "_doc": { + "properties": { + "my_vector": { + "type": "sparse_vector" + }, + "my_text" : { + "type" : "keyword" + } + } + } + } + } + + PUT sindex/_doc/1 + { + "my_text" : "text1", + "my_vector" : {"1": 0.5, "99": -0.5, "5": 1} + } + +commit 9c1cdea8391e78f3d8ced72c4164bc28f4f593bc +Author: Tal Levy +Date: Wed Dec 12 16:28:11 2018 -0800 + + [cleanup] remove deprecated references to dataWithClusterFiles (#36574) + + data files under the cluster name subdirectory has been deprecated and was + meant to be removed in 6.0. This commit removes some leftover referrences to + these paths. + +commit 6a824322fc5f5b96fdd211be94690ad7fdff33ff +Author: Gordon Brown +Date: Wed Dec 12 14:57:48 2018 -0700 + + Improve error message for deleting in-use policy (#36457) + + The error message used when attempting to delete a lifecycle policy that + is in use previously only included one index which was using the policy. + It now includes all indices using that policy. + +commit 71a39d10be8b33d63dcb12088ebf2ae81124dd6d +Author: Julie Tibshirani +Date: Wed Dec 12 12:57:32 2018 -0800 + + Make sure that BWC tests run successfully, even with types deprecation messages. (#36511) + +commit 9ac73596431cb8fabcd26855c0c1861a435f5f34 +Author: Alan Woodward +Date: Wed Dec 12 20:18:32 2018 +0000 + + Update lucene to snapshot-7e4555a2fd (#36563) + + Includes the following: + + * Reversion of doc-values changes in LUCENE-8374; we are interested in seeing if this + has an effect on benchmarks for node-stats and index-stats + * More improvements to docvalues updates + +commit 4a8cd45cca3fb485ef916f7ee5d65f190976cf01 +Author: Jason Tedor +Date: Wed Dec 12 15:11:28 2018 -0500 + + Avoid blocking non-reproducible randomness in test (#36561) + + The security documentation test uses + SecureRandom#getStrongInstance. This defaults to + securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN which + means a blocking implementation that reads from /dev/random. This means + that this test can stall if the entropy on the machine is + exhausted. Anyway, it also means that the randomness is + non-reproducible, a thing that we try to avoid in tests. This commit + switches to a boring randomness source to avoid the blocking, and to + keep the test reproducible. + +commit 33152f648fce649b774fdb659bc47a7134755c60 +Author: Julie Tibshirani +Date: Wed Dec 12 10:38:02 2018 -0800 + + Fix some inconsistencies in the types deprecation code. (#36517) + + * Make sure to test conversion for both typed and typeless HLRC requests. + * Update a few more statements to deprecatedAndMaybeLog. + * Make sure Rest*SearchTemplateActionTests extend RestActionTestCase. + +commit 7f612d5dd86223f5c0cb0b7c9ff6385c5075af59 +Author: Tim Brooks +Date: Wed Dec 12 09:39:15 2018 -0700 + + Always compress based on the settings (#36522) + + Currently TransportRequestOptions allows specific requests to request + compression. This commit removes this and always compresses based on the + settings. Additionally, it removes TransportResponseOptions as they + are unused. + + This closes #36399. + +commit 02d0f163a4017a58f945fffef91183d7b2fec496 +Author: Armin Braun +Date: Wed Dec 12 16:48:34 2018 +0100 + + MINOR: Cleanup Dead Code in Snapshots Package (#36528) + +commit dbcafbefc9c4adf8013dd0a3d552926e86b8e23a +Merge: 593d8637c48 ff5dd14753c +Author: Tanguy Leroux +Date: Wed Dec 12 16:45:31 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit ff5dd14753c2588f3be1ac16e3adb4fe38c3f590 +Author: Simon Willnauer +Date: Wed Dec 12 16:21:02 2018 +0100 + + Fix test failures related to file corruption (#36530) + + * Fix CorruptFileIT to also take last DV generation into account + + We currently only prune old .liv generations. With soft_deletes it's important + to also prune DV generations. + + * Fix CorruptionUtils to skip the footer bytes after the checksum is read. + + Today we read a broken checksum since we also checksum the 8 footer bytes that include + the checksum algorithm and the footer magic. + + Closes #36526 + +commit 3a56bb0924cd46cc55839c1aa7c4679f72e70c9e +Author: Tim Brooks +Date: Wed Dec 12 07:55:17 2018 -0700 + + Do not zero out pages for bytes output stream (#36523) + + Currently, the `BytesStreamOutput` always zeros out the underlying byte + pages when they are acquired. This should not be necessary as the stream + overwrites the underlying bytes as serialization occurs. + +commit e63d52af63ed66bbc8400935efc4c359f84b5a08 +Author: Tim Brooks +Date: Wed Dec 12 07:00:50 2018 -0700 + + Move page size constants to PageCacheRecycler (#36524) + + `PageCacheRecycler` is the class that creates and holds pages of arrays + for various uses. `BigArrays` is just one user of these pages. This + commit moves the constants that define the page sizes for the recycler + to be on the recycler class. + +commit bdb1e0e04e29054b0ca5a2d108914c7deed4c86e +Author: Jason Tedor +Date: Wed Dec 12 08:50:21 2018 -0500 + + Make node field in JoinRequest private (#36405) + + This commit moves the node field in the JoinRequest object to be a + private field, adding a dedicated accessor. This is a minor breaking + change in that it is no longer possible for all callers to overwrite + this field, but that is a feature. + +commit dabe29e7837207ff4dbcb3d1748db3a83b7e0951 +Author: Max Penet +Date: Wed Dec 12 13:57:53 2018 +0100 + + [Docs] Adding community clojure client built on java low level rest client (#36484) + +commit aa43e0b2ccfa2bd4c7b367b1bf9c7d0434216f7c +Author: David Turner +Date: Wed Dec 12 12:52:34 2018 +0000 + + [Zen2] Migrate no-master-block integration tests (#36502) + + This change follows up on #36478 by migrating the affected integration tests to + use Zen2. + +commit 883940ad9243b03d923cb9c01588ffc510efceee +Author: Martijn van Groningen +Date: Wed Dec 12 12:47:24 2018 +0100 + + [CCR] Change AutofollowCoordinator to use wait_for_metadata_version (#36264) + + Changed AutofollowCoordinator makes use of the wait_for_metadata_version + feature in cluster state API and removed hard coded poll interval. + + Originates from #35895 + Relates to #33007 + +commit c6de68c3b90db307786bd6872d379f6461c3f788 +Author: Tim Vernum +Date: Wed Dec 12 21:35:56 2018 +1100 + + Fix realm config in SAML QA test (#36515) + + An issue was introduced due to the merge of authorization_realms with + the change to use Affix Settings for realms. + The ".type" setting no longer exists as the type is now part of the + setting key. + +commit 562f09603beca17100bdc7b65d64e183702383c1 +Author: Alpar Torok +Date: Wed Dec 12 12:24:39 2018 +0200 + + plugin install: don't print download progress in batch mode (#36361) + + * Don't print download progress in batch mode + + With this change we will no longer provide the progress bar in batch + mode. + Assuming that this is mode is mainly for consumption by tools which + will serialize the output, we shouldn't print a progress bar to be + for every percentile. + + * PR review + +commit f920f571d1f7297cdcc12a4403b94ab2c803556f +Author: Alpar Torok +Date: Wed Dec 12 12:24:17 2018 +0200 + + Fix possible missing class error (#36491) + + - Use the `testRuntime` classpath when loading the tests instead of + compile + - simplify task action + - improove error message when classes are missing + +commit 09a9e2236ce4ccff9b3bb0b5d2a57b7b4c229912 +Author: Alpar Torok +Date: Wed Dec 12 12:04:36 2018 +0200 + + Third party audit improovements (#36167) + + - fix up to date checks to ignore elasticsearch jars. We were not scanning them but these still triggered a rebuild. + - add tests to assert correct behavior and up to date checks. + - make the task less verbose with `-i` and include the output only on errors. + +commit c00d0fc814d8c0732c965905e00ae49eb3f4be7d +Author: Alpar Torok +Date: Wed Dec 12 12:00:47 2018 +0200 + + Test fixtures improovements (#36037) + + * Upgrae plugin to latest and expose udp + * Explicit check for windows + * Rename the properties for the port numbers + * Tasks for pre and pos container actions + +commit 4a825e2e86582fa2d693d64246c2fb6669f42fdb +Author: Martijn van Groningen +Date: Wed Dec 12 09:55:37 2018 +0100 + + [CCR] Clean followed leader index UUIDs in auto follow metadata (#36408) + + The auto follow coordinator keeps track of the UUIDs of indices that it has followed. The index UUID strings need to be cleaned up in the case that these indices are removed in the remote cluster. + + Relates to #33007 + +commit c4f437800680a8858131b5a5e7bb370c41d109a9 +Author: Ryan Ernst +Date: Tue Dec 11 23:47:44 2018 -0800 + + Core: Rework multi date formatter merging (#36447) + + This commit moves the MergedDateFormatter to a package private class and + reworks joda DateFormatter instances to use that instead of a single + DateTimeFormatter with multiple parsers. This will allow the java and + joda multi formats to share the same format parsing method in a + followup. + +commit 1bb6f844fee92d1c240a124499f0c299c09a1daf +Author: Nhat Nguyen +Date: Wed Dec 12 02:03:23 2018 -0500 + + AwaitsFix testCorruptFileThenSnapshotAndRestore + + Tracked at #36526 + +commit 3fb5a12b305a5926e705d77cc14e8abdd4f937f6 +Author: Nhat Nguyen +Date: Wed Dec 12 00:47:40 2018 -0500 + + Upgrade to Lucene-8.0.0-snapshot-61e448666d (#36518) + + Includes: + - LUCENE-8602: Share TermsEnum if possible while applying DV updates + +commit 143f151185c01698d9416de2a530714c40d9b96e +Author: Tim Vernum +Date: Wed Dec 12 15:12:49 2018 +1100 + + HLRC: Implement get-user-privileges API (#36292) + + This adds the _security/user/_privileges API to the High + Level Rest Client. + + This also makes some changes to the Java model for the Role APIs + in order to better accommodate the GetPrivileges API + +commit 03daad9812b6469941e5597edd3060e2c78f3a79 +Author: Nik Everett +Date: Tue Dec 11 19:43:17 2018 -0500 + + Re-deprecate xpack rollup endpoints (#36451) + + Redeprecates the `/_xpack/rollup` endpoints in favor of `/_rollup`. + + When we cleanup the rollup in a cluster containing 6.x nodes we need to + use `/_xpack/rollup` instead of `/_rollup` because the 6.x nodes don't + know about `/_rollup`. In those cases we must ignore the deprecation + warnings that the 7.0 node will return for the end point. + + Closes #36044 + +commit de373060fb7a468419f41cffc4a6a3a2e5dc89c2 +Author: Andrei Stefan +Date: Wed Dec 12 02:29:02 2018 +0200 + + SQL: non ISO 8601 versions of DAY_OF_WEEK and WEEK_OF_YEAR functions (#36358) + + * Renamed DAY_OF_WEEK and WEEK_OF_YEAR functions to their ISO version and + added the same functions with different functionality. + * Rewritten the datetime functions documentation to follow the format of the other + functions documentation pages. + +commit 51800de2a8d21fe970eadcc3197fbaa9b0245775 +Author: Nhat Nguyen +Date: Tue Dec 11 18:58:49 2018 -0500 + + Enable soft-deletes by default on 7.0.0 or later (#36141) + + This change enables soft-deletes by default on ES 7.0.0 or later. + + Relates #33222 + + Co-authored-by: Jason Tedor + +commit 797f985067ea711ee2c3a0120347421af592eac4 +Author: Tim Brooks +Date: Tue Dec 11 16:09:28 2018 -0700 + + Add version to handshake requests (#36171) + + Currently our handshake requests do not include a version. This is + unfortunate as we cannot rely on the stream version since it is not the + sending node's version. Instead it is the minimum compatibility version. + The handshake request is currently empty and we do nothing with it. This + should allow us to add data to the request without breaking backwards + compatibility. + + This commit adds the version to the handshake request. Additionally, it + allows "future data" to be added to the request. This allows nodes to craft + a version compatible response. And will properly handle additional data in + future handshake requests. The proper handling of "future data" is useful + as this is the only request where we do not know the other node's version. + + Finally, it renames the TcpTransportHandshaker to + TransportHandshaker. + +commit 55743aac470ea97a4c9c217f2b3a083dfefa57de +Author: lcawl +Date: Tue Dec 11 14:24:47 2018 -0800 + + [DOCS] Adds placeholder for alpha2 release notes + +commit 2f18325384c77fc10f86847bcbaf268626839ce7 +Author: Mayya Sharipova +Date: Tue Dec 11 17:09:59 2018 -0500 + + Deprecate types in update_by_query and delete_by_query (#36365) + + Relates to #35190 + +commit 8e988f6c06a4f55fbc52b910ef4f691a651f36f7 +Author: Jack Conradson +Date: Tue Dec 11 14:06:38 2018 -0800 + + [Painless] Add def to boxed type casts (#36506) + + This adds casts for the def type to all standard boxed types. Prior to this certain + casts such as def [long/Long] -> Double would fail which does not follow the + goals of the Painless casting model to remove the need for explicit boxing. This + also creates symmetry with the casts for the newly created bridge methods + being called at run-time. + +commit c1e3bdd42d710838722919ae07ab75d246791ee5 +Author: docteurjay +Date: Tue Dec 11 16:17:15 2018 -0500 + + docs: Remove issue key from jira documentation (#36136) + + This commit removes an incorrect "issues" key from the documentation. + see: https://developer.atlassian.com/server/jira/platform/rest-apis/ + + Fixes #36128 + +commit f23701406b6479fa7470685275112d9a310a1c76 +Author: Nhat Nguyen +Date: Tue Dec 11 15:00:09 2018 -0500 + + CCR/TEST: Enable soft-deletes in ShardChangesActionTests + + Relates #36446 + +commit f2e18a6bf6b5fa24213b3ce7d4e2bc76e1d08aca +Author: Jason Tedor +Date: Tue Dec 11 14:33:53 2018 -0500 + + Add check for minimum required Docker version (#36497) + + Our Docker build uses a multi-stage Docker build. This requires Docker + version 17.05 or greater. Without an explicit check here, the build + fails in a mysterious way such as "invalid reference format" that is + hard to track down (Google searches for "Docker invalid reference + format" do not turn up anything useful). This commit refactors our + existing Docker checks, and adds a new one for the minimum Docker + version. + +commit dafea3cc23cdc1694843ee1be310a149ad027c10 +Author: Luca Cavanna +Date: Tue Dec 11 20:24:29 2018 +0100 + + Minor search controller changes (#36479) + + This commit contains a few minor changes to our search code: + + - adjust the visibility of a couple of methods in our search code to package private from public or protected. + - make some of the `SearchPhaseController` methods static where possible + - rename one of the `SearchPhaseController#reducedQueryPhase` methods (used only for scroll requests) to `reducedScrollQueryPhase` without the `isScrollRequest` argument which was always set to `true` + - replace leniency in `SearchPhaseController#setShardIndex` with an assert to make sure that we never set the shard index twice + - remove two null checks where the checked field can never be null + - resolve an unchecked warning + - replace `List#toArray` invocation that creates an array providing the true size with array creation of length 0 + - correct a couple of typos in comments + +commit fb18b35347c8283bb72b6cb10aa83b9f2e7bb9f3 +Author: Luca Cavanna +Date: Tue Dec 11 20:19:01 2018 +0100 + + Add default methods to DocValueFormat (#36480) + + The different `DocValueFormat` implementors throw `UnsupportedOperationException` for methods that they don't support. That is perfectly fine, and quite common as not all implementors support all of the possible formats. This makes it hard though to trace back which implementors support which formats as they all implement the same methods. + + This commit introduces default methods in the `DocValueFormat` interface so that all methods throw `UnsupportedOperationException` by default. This way implementors can override only the methods that they specifically support. + +commit eb733f404aa9ce97a5c2fe6a6980521a9c455c74 +Author: Julie Tibshirani +Date: Tue Dec 11 10:06:54 2018 -0800 + + Fix the mixed cluster REST test explain/11_basic_with_types. + +commit 790f8102e9c808ec1e497f77bb0e93dc5534d346 +Author: Tim Brooks +Date: Tue Dec 11 11:55:41 2018 -0700 + + Modify `BigArrays` to take name of circuit breaker (#36461) + + This commit modifies BigArrays to take a circuit breaker name and + the circuit breaking service. The default instance of BigArrays that + is passed around everywhere always uses the request breaker. At the + network level, we want to be using the inflight request breaker. So this + change will allow that. + + Additionally, as this change moves away from a single instance of + BigArrays, the class is modified to not be a Releasable anymore. + Releasing big arrays was always dispatching to the PageCacheRecycler, + so this change makes the PageCacheRecycler the class that needs to be + managed and torn-down. + + Finally, this commit closes #31435 be making the serialization of + transport messages use the inflight request breaker. With this change, + we no longer push the global BigArrays instnace to the network level. + +commit a07fb312fc17c2ca48e9462fd3d1c3e528648f26 +Author: James Baiera +Date: Tue Dec 11 13:49:24 2018 -0500 + + Move LoggedExec to minimumRuntime source set (#36453) + + This moves LoggedExec to the minimumRuntime source set so that it correctly + compiles for earlier Java versions, namely Java 8. + +commit 405a38131b6bc8da898c2b5014991ab04d6a0a1b +Author: Martijn van Groningen +Date: Tue Dec 11 19:38:40 2018 +0100 + + added 6.5.4 version + +commit c595a73c707f070cf78fae04916c79d97fa466c6 +Author: Christoph Büscher +Date: Tue Dec 11 19:07:22 2018 +0100 + + Add test logging for #35644 + + Adding logging to confirm the suspicion that there might be another old index + lingering around in the test cluster in cases where this test occasionally + fails. + +commit a9eccbcd02299e036ec3cad50848c67428ad631e +Author: markharwood +Date: Tue Dec 11 17:30:15 2018 +0000 + + Tests- added helper methods to ESRestTestCase for checking warnings (#36443) + + Added helper methods to ESRestTestCase for checking warnings in mixed and current-version-only clusters. + This is supported by a new VersionSpecificWarningsHandler class with associated unit test. + Closes #36251 + +commit eead8a144b27df0e311f84086381ce8823380054 +Author: Andrei Stefan +Date: Tue Dec 11 19:17:51 2018 +0200 + + SQL: move requests' parameters to requests JSON body (#36149) + +commit 21d91f126889082bf67e4b13575b2c9fbdbfaf32 +Author: David Turner +Date: Tue Dec 11 16:40:19 2018 +0000 + + [Zen2] Respect the no_master_block setting (#36478) + + Today the Zen2 coordinator only applies a write block when there is no known + elected master, ignoring the `discovery.zen.no_master_block` setting. This + commit resolves this, applying the correct block according to the configuration + instead. + +commit 084e06e481bc14f0ec1a71cc0444b1e05a1bbe20 +Author: Nhat Nguyen +Date: Tue Dec 11 11:18:10 2018 -0500 + + Require soft-deletes when access changes snapshot (#36446) + + Today we do not enforce soft-deletes when accessing the Lucene changes + snapshot. This might lead to internal errors because we assume + soft-deletes are enabled in that code path. + +commit 8b821706cc901cecdc7e9758f4074384bdd9896d +Author: Andrey Ershov +Date: Tue Dec 11 17:12:17 2018 +0100 + + Switch more tests to zen2 (#36367) + + 1. CCR tests work without any changes + 2. `testDanglingIndices` require changes the source code (added TODO). + 3. `testIndexDeletionWhenNodeRejoins` because it's using just two + nodes, adding the node to exclusions is needed on restart. + 4. `testCorruptTranslogTruncationOfReplica` starts dedicated master + one, because otherwise, the cluster does not form, if nodes are stopped + and one node is started back. + 5. `testResolvePath` needs TEST cluster, because all nodes are stopped + at the end of the test and it's not possible to perform checks needed + by SUITE cluster. + 6. `SnapshotDisruptionIT`. Without changes, the test fails because Zen2 + retries snapshot creation as soon as network partition heals. This + results into the race between creating snapshot and test cleanup logic + (deleting index). Zen1 on the + other hand, also schedules retry, but it takes some time after network + partition heals, so cleanup logic executes latter and test passes. The + check that snapshot is eventually created is added to + the end of the test. + +commit 13b1f197724cffea475b8fefbe2bb3df343c92e2 +Author: Jack Conradson +Date: Tue Dec 11 08:03:08 2018 -0800 + + [Painless] Add extensive tests for def to primitive casts (#36455) + + This adds tests for each possible cast of def to a primitive type both implicit + and explicit. This also fixes a minor bug where we were only checking the type + of a def to be Number in some explicit casts. This does not work because it + allows possible unintended casts from BigInteger and BigDecimal to primitive + types since they both extend Number but are not included as part of the + Painless casting model. + +commit 6481f2e380c18eb98f0d946fc6a8c11bce655697 +Author: Gordon Brown +Date: Tue Dec 11 08:53:05 2018 -0700 + + Add setting to bypass Rollover action (#36235) + + Adds a setting that indicates that an index is done indexing, set by ILM + when the Rollover action completes. This indicates that the Rollover + action should be skipped in any future invocations, as long as the index + is no longer the write index for its alias. + + This enables 1) an index with a policy that involves the Rollover action + to have the policy removed and switched to another one without use of + the move-to-step API, and 2) integrations with Beats and CCR. + +commit c556b9ef191d660c6b4e820cdc427481eecd3eb4 +Author: Jason Tedor +Date: Tue Dec 11 10:48:14 2018 -0500 + + Try running CI against Zulu (#36391) + + This commit adds Zulu to the CI rotation. This commit only means that we + are going to try running CI against Zulu, it does not carry any + implications of support in the past, now, or in the future. + +commit c9e03e6ead6933a4ca96a47f7995ea64a7b1850c +Author: debadair +Date: Tue Dec 11 07:44:57 2018 -0800 + + [DOCS] Reworked the shard allocation filtering info. (#36456) + + * [DOCS] Reworked the shard allocation filtering info. Closes #36079 + + * Added multiple index allocation settings example back. + + * Removed extraneous space + +commit c3a6d1998a79cf2d7d72d3ff579a6a18975599e8 +Author: David Turner +Date: Tue Dec 11 12:53:05 2018 +0000 + + Log [initial_master_nodes] on formation failure (#36466) + + Today we log a slightly cryptic "cluster bootstrapping is disabled on this + node" message if bootstrapping hasn't been configured. Since there is today + only one way to bootstrap the cluster it seems preferable to spell out exactly + which setting is missing. + +commit 00eadd93eb646ddc1e387e7fd042498646a720f8 +Author: Christian Schneider +Date: Tue Dec 11 06:15:44 2018 -0600 + + converting ForbiddenPatternsTask to .java (#36194) + + * converting ForbiddenPatternsTask to java impl & unit tests + +commit 5f7524bb89e8ec0c761f54fbe563497b395e0b8c +Author: Martijn van Groningen +Date: Tue Dec 11 13:13:17 2018 +0100 + + fixed typo + +commit 031d0566b0bbf33e47f9914da1f68d5abb394668 +Author: Marios Trivyzas +Date: Tue Dec 11 12:29:44 2018 +0200 + + SQL: [docs] Add `beta[]` markup to all SQL pages (#36471) + + Closes: #36437 + +commit d01b30acba3b015a14d883e44eb68e66717717ee +Author: Yu +Date: Tue Dec 11 18:30:58 2018 +0800 + + lower fielddata circuit breaker's default limit (#27162) + + * Lower fielddata circuit breaker default limit + + Lower fielddata circuit breaker default limit from 60% to 40% as we have + moved to doc_values for most of the cases. + + * merge master in + + * update tests + + * update docs + +commit c635904a1b41aee3b6fca639369d426d019b15d6 +Author: Martijn van Groningen +Date: Tue Dec 11 11:28:25 2018 +0100 + + muted mixed clusters explain/11_basic_with_types/Explain body without query element} test + +commit 71cf6faaad55319e937700824654e49f128ba696 +Author: Yannick Welsch +Date: Tue Dec 11 11:22:23 2018 +0100 + + Only turn to follower when term bumping on follower check (#36449) + + Deals with a situation where a follower becomes disconnected from the leader, but only for such a + short time where it becomes candidate and puts up a NO_MASTER_BLOCK, but then receives a + follower check from the leader. If the leader does not notice the node disconnecting, it is important + for the node not to be turned back into a follower but try and join the leader again. + + We still should prefer the node into a follower on a follower check when this follower check triggers + a term bump as this can help during a leader election to quickly have a leader turn all other nodes + into followers, even before the leader has had the chance to transfer a possibly very large cluster + state. + + Closes #36428 + +commit d8e3d97a7deb44aabca9d2313124dab9b82d3c6f +Author: Yannick Welsch +Date: Tue Dec 11 10:35:58 2018 +0100 + + Add discovery types to cluster stats (#36442) + + Adds information about the used discovery types to the cluster stats, similar as we have for the network types. + +commit d7c5d8049afa2edea2cf13c1ec423647bea6cae5 +Author: Ioannis Kakavas +Date: Tue Dec 11 11:13:10 2018 +0200 + + Deprecate /_xpack/security/* in favor of /_security/* (#36293) + + * This commit is part of our plan to deprecate and ultimately remove the use of _xpack in the REST APIs. + + - REST API docs + - HLRC docs and doc tests + - Handle REST actions with deprecation warnings + - Changed endpoints in rest-api-spec and relevant file names + +commit bf42cedc138b041e66d8409d5b4f60a29660d75f +Author: David Turner +Date: Tue Dec 11 09:02:23 2018 +0000 + + Clarify the order of responses to a bulk request (#36441) + + Bulk requests comprise many individual actions, and the responses for each + action comes back in the same order (see e.g. `DocumentActionsIT#testBulk()`). + However the docs do not seem to explicitly state this vital fact. This commit + addresses that omission. + +commit 2bf0583fec45625aa76d83c042cb0fbeb53a8c99 +Author: David Turner +Date: Tue Dec 11 09:02:01 2018 +0000 + + Exclude Zen2 fault detection traffic from tracer (#36444) + + Today we exclude `internal:discovery/zen/fd*` from the transport tracer by + default. This commit excludes the new fault detection mechanism similarly. + +commit 87831051dcd9d21a79411b78628d7d1f7f96fa74 +Author: Julie Tibshirani +Date: Mon Dec 10 19:45:13 2018 -0800 + + Deprecate types in explain requests. (#35611) + + The following updates were made: + - Add a new untyped endpoint `{index}/_explain/{id}`. + - Add deprecation warnings to Rest*Action, plus tests in Rest*ActionTests. + - For each REST yml test, make sure there is one version without types, and another legacy version that retains types (called *_with_types.yml). + - Deprecate relevant methods on the Java HLRC requests/ responses. + - Update documentation (for both the REST API and Java HLRC). + +commit a0da390df2c2e6a4881e67e8038186344cb94676 +Author: Ryan Ernst +Date: Mon Dec 10 17:29:25 2018 -0800 + + Scripting: Switch watcher to use joda bwc time objects (#35966) + + This commit converts the watcher execution context to use the joda + compat java time objects. It also again removes the joda methods from + the painless whitelist. + +commit 99f89cd3b456c086986b4aef3a6b0dc401d17005 +Author: Julie Tibshirani +Date: Mon Dec 10 17:22:48 2018 -0800 + + Deprecate types in get, exists, and multi get. (#35930) + + For each API, the following updates were made: + - Add deprecation warnings to `Rest*Action`, plus tests in `Rest*ActionTests`. + - For each REST yml test, make sure there is one version without types, and another legacy version that retains types (called *_with_types.yml). + - Deprecate relevant methods on the Java HLRC requests/ responses. + - Update documentation (for both the REST API and Java HLRC). + +commit b030125e23acbd2326708219a2af694694199bc5 +Author: Jason Tedor +Date: Mon Dec 10 18:36:55 2018 -0500 + + Build Docker images if the binary exists (#36459) + + This commit changes the Docker assemble tasks so that they attempt to + build Docker if the Docker binaries exist, of if build.docker is set to + true. If the Docker binaries do not exist, or if build.docker is set to + false, then no attempt is made to build the Docker images. + +commit de06769677bc0ecb503adfef9e11bd880b88ecab +Author: Nik Everett +Date: Mon Dec 10 16:29:43 2018 -0500 + + Docs: Fix link to transport client + + The transport client was linking to a page in maven central that didn't + have the transport client in it. This is better link and happens to be + the same link that the low and high level rest client use. + +commit 2a7edca59ffcff797689694f30334c30b5498663 +Author: Nhat Nguyen +Date: Mon Dec 10 16:33:49 2018 -0500 + + Upgrade to Lucene-8.0.0-snapshot-ef61b547b1 (#36450) + + Includes: + + - LUCENE-8598: Improve field updates packed values + - LUCENE-8599: Use sparse bitset to store docs in SingleValueDocValuesFieldUpdates + +commit d615add1b16655c0d30b50e800922a316bbecd17 +Author: Jernej Klancic +Date: Mon Dec 10 14:02:49 2018 -0700 + + Add pipeline parent validation for auto date histogram (#35670) + + Allow `auto_date_histogram` as a valid parent agg for derivative, + cumulative sum, moving average, moving function and serial + differencing pipeline aggregations. + + Since all these aggs share the same requirement (sequentially + ordered parent aggs), this commit also refactors to share + the same validation code so that any newly added aggs won't + be forgotten. + + Closes #35578 + +commit 01afeff55d1f8d490ab26fce1e9ae3645bb90025 +Author: Albert Zaharovits +Date: Mon Dec 10 21:54:47 2018 +0200 + + Fix origin.type for connection_* audit events (#36410) + + The `origin.type` field's permitted values are now `rest` or + `transport` (as the docs declare) instead of `ip_filter`. + +commit 0909a631ba7ee62555fdd5a9e56e224338407d14 +Author: Jason Tedor +Date: Mon Dec 10 14:50:30 2018 -0500 + + Add non-X-Pack centric rollup endpoints (#36383) + + * Add non-X-Pack centric rollup endpoints + + This commit adds new endpoints for rollup that do not have _xpack in + their path. The purpose for this change is to take these endpoints into + 6.x as well so that they can be available in mixed cluster tests too. A + follow-up change will deprecate the use of _xpack in the rollup + endpoints. And finally, in the future, we would remove the _xpack + endpoints. + + * Remove import + + * Fix typo + +commit 75392adf60b011e8faae3d8cb18da158f84f7c26 +Author: Jim Ferenczi +Date: Mon Dec 10 20:41:20 2018 +0100 + + [TEST] Convert SearchHitsTests to AbstractStreamableXContentTestCase (#36313) + + This change adds a way to provide the content type of the rest serialization + tests when creating random instances. This is used by SearchHitsTests to ensure + that the internal members of the class are created with the same xContentType + and that equals can be used to compare an instances created from an XContent + view. + +commit ed7afd1a9e972ad46b5739219d65db7fe77addfb +Author: Tal Levy +Date: Mon Dec 10 09:28:17 2018 -0800 + + [ILM] TEST: fix long overflow in TimeValueScheduleTests (#36384) + + Closes #35948. + +commit 58a5ad1f1e344813b04ca53254c4d725ae422d52 +Author: Gordon Brown +Date: Mon Dec 10 10:23:58 2018 -0700 + + Add Tribe removal to breaking changes list (#36239) + +commit ca3f5c1e2e7454ed663d57c4b90aa6db5103c985 +Author: David Turner +Date: Mon Dec 10 17:23:03 2018 +0000 + + Cancel GetDiscoveredNodesAction when bootstrapped (#36423) + + Today the `GetDiscoveredNodesAction` waits, possibly indefinitely, to discover + enough nodes to bootstrap the cluster. However it is possible that the cluster + forms before a node has discovered the expected collection of nodes, in which + case the action will wait indefinitely despite the fact that it is no longer + required. + + This commit changes the behaviour so that the action fails once a node receives + a cluster state with a nonempty configuration, indicating that the cluster has + been successfully bootstrapped and therefore the `GetDiscoveredNodesAction` + need wait no longer. + + Relates #36380 and #36381; reverts 558f4ec27820e1a50660dc1f3437422150339af0. + +commit c3dd0d393d479496360eb0187727d61651ff89b1 +Author: Marios Trivyzas +Date: Mon Dec 10 18:52:09 2018 +0200 + + SQL: Simplify function registration and resolution (#36417) + + Previously, we used a CamelCase to CAMEL_CASE transformation to get the + primary name of a function from its class name which led to some issues + since there are functions that we don't want to be registered this way + (e.g.: IFNULL). To simplify the logic and avoid and "magic" + transformations in the FunctionRegistry a primary name must be provided + explicitely for each function. + + The same change is applied for the function resolution (when a function + is used in an SQL statement). There is no CamelCase to CAMEL_CASE + transformation but only upper-casing is applied (fuNcTiOn -> FUNCTION). + +commit 10938e5135b381c002e300835806f185ac28a9e2 +Author: Andy Bristol +Date: Mon Dec 10 08:11:57 2018 -0800 + + [hlrc] add index templates exist API (#36132) + + This commit adds support for the index templates exist API, creating + new client-side request types for that API and the get index + templates API. Also adds links in hlrc docs to pages for supported + index template APIs + +commit 1a36046421cb562f49376aee0baed6c84e1746f3 +Author: Ryan Ernst +Date: Mon Dec 10 08:06:41 2018 -0800 + + Core: Convert FormatDateTimeFormatter to DateFormatter (#36396) + + This commit creates JodaDateFormatter to replace + FormatDateTimeFormatter. It converts all uses of the old class + to DateFormatter to allow a future change to use JavaDateFormatter + when appropriate. + +commit 891a37adaa4804dd2cc5fc060d864bb29e593efe +Author: Dimitris Athanasiou +Date: Mon Dec 10 15:05:47 2018 +0000 + + [ML] Use LinkedHashMap to preserve order in job and datafeed fields (#36389) + +commit 9626e700ceab6185ac7ef33934fed28cb87aaa26 +Author: Nik Everett +Date: Mon Dec 10 08:32:00 2018 -0500 + + LLRC: Make warning behavior pluggable per request (#36345) + + This allows you to plug the behavior that the LLRC uses to handle + warnings on a per request basis. + + We entertained the idea of allowing you to set the warnings behavior to + strict mode on a per request basis but that wouldn't allow the high + level rest client to fail when it sees an unexpected warning. + + We also entertained the idea of adding a list of "required warnings" to + the `RequestOptions` but that won't work well with failures that occur + *sometimes* like those we see in mixed clusters. + + Adding a list of "allowed warnings" to the `RequestOptions` would work + for mixed clusters but it'd leave many of the assertions in our tests + weaker than we'd like. + + This behavior plugging implementation allows us to make a "required + warnings" option when we need it and an "allowed warnings" behavior when + we need it. + + I don't think this behavior is going to be commonly used by used outside + of the Elasticsearch build, but I expect they'll be a few commendably + paranoid folks who could use this behavior. + +commit f79e602d9f8f6d9c4e2e55c573511e3f8d3ee5ec +Author: Yannick Welsch +Date: Mon Dec 10 13:53:09 2018 +0100 + + Mute ClusterDisruptionIT.testSendingShardFailure + + Relates to #36428 + +commit 38a1d4fcdbaefa2ef290d3495a114763b4187c4f +Author: Martijn van Groningen +Date: Mon Dec 10 13:15:15 2018 +0100 + + re-enabled test with added logging to have more info when it fails. + +commit ca916f7186b263e40b658515ca501581a9121ec6 +Author: Martijn van Groningen +Date: Mon Dec 10 13:05:47 2018 +0100 + + [TEST] pause index following after test + +commit c9c691b0cfe078f5fce7398ee72996f3dafca757 +Author: David Pilato +Date: Mon Dec 10 12:44:39 2018 +0100 + + Change Lucene Snapshot example for 7.0.0 (#35619) + + Update to a more recent version our example. + +commit 558f4ec27820e1a50660dc1f3437422150339af0 +Author: David Roberts +Date: Mon Dec 10 11:07:02 2018 +0000 + + Ignore zen2 discovery task in waitForPendingTasks (#36381) + + Fixes #36380 + +commit 0d9e3adc951ca5b5d7978ba7eb32cdb1d37e5fa3 +Author: Igor Suhorukov +Date: Mon Dec 10 13:26:24 2018 +0300 + + Loop replace with Collection.removeIf() (#36351) + +commit 6e6e63d01da8fab85d3a69656416c39bfd8c9139 +Author: Yannick Welsch +Date: Mon Dec 10 11:00:57 2018 +0100 + + Zen2: Move all mixed-version REST tests to Zen2 (#36398) + + Moves all remaining (rolling-upgrade and mixed-version) REST tests to use Zen2. To avoid adding + extra configuration, it relies on Zen2 being set as the default discovery type. This required a few + smaller changes in other tests. I've removed AzureMinimumMasterNodesTests which tests Zen1 + functionality and dates from a time where host providers were not configurable and each cloud + plugin had its own discovery.type, subclassing the ZenDiscovery class. I've also adapted a few tests + which were unnecessarily adding addTestZenDiscovery = false for the same legacy reasons. Finally, + this also moves the unconfigured-node-name REST test to Zen2, testing the auto-bootstrapping + functionality in development mode when no discovery configuration is provided. + +commit 0b1e7e96becf992bb2707bf90822070073949130 +Author: Albert Zaharovits +Date: Mon Dec 10 11:54:50 2018 +0200 + + Nitfix for SecurityDocumentationIT dad6f1c9fe + +commit dad6f1c9fe9b25ed3ee2d16c1112a5ef3f5f908a +Author: Albert Zaharovits +Date: Mon Dec 10 09:41:31 2018 +0200 + + [HLRC] Put Role (#36209) + + This commit adds support for the put role API in the + java high level rest client. + +commit bf2c61d2d0d1d495b2992c45549c80b5f1e085d3 +Author: Alpar Torok +Date: Mon Dec 10 08:32:19 2018 +0200 + + Forbid empty testing tasks (#36259) + + Closes #34820 + + With this change we allow for no tests being ran in randomized testing + task, and forbid empty testing tasks from the testing conventions task. + We will no longer have to disable the task if all tests are muted. + +commit e160346b9af5bb15805c89f7c4ee2904d57af430 +Author: Alpar Torok +Date: Mon Dec 10 08:23:03 2018 +0200 + + Improve logged exec output readability (#36217) + + * Improve logged exec output readability + + - Split error and out streams and log them separately + - Log everything in a single call to prevent interference from + other log messages + +commit a42502df8b81e3035c716c4c6a83c9e05abefff4 +Author: Christoph Büscher +Date: Mon Dec 10 01:00:42 2018 +0100 + + [Docs] Add description of simple query string flags (#36211) + + Closes #34944 + +commit f4aac8d3f17ab92d03166b9ae4b0aeb682f06d3f +Author: Martijn van Groningen +Date: Sun Dec 9 19:59:08 2018 +0100 + + [HLRC] Added support for Follow Stats API (#36253) + + This change also adds documentation for the Follow Stats API. + + Relates to #33824 + +commit 633ab24017f44164bc604f25ef7524723180a50e +Author: Martijn van Groningen +Date: Sun Dec 9 19:34:48 2018 +0100 + + [CCR] Restructured QA modules (#36404) + + Renamed the follow qa modules: + `multi-cluster-downgraded-to-basic-license` to `downgraded-to-basic-license` + `multi-cluster-with-non-compliant-license` to `non-compliant-license` + `multi-cluster-with-security` to `security` + + Moved the `chain` module into the `multi-cluster` module and + changed the `multi-cluster` to start 3 clusters. + + Followup from #36031 + +commit bba9bb2cfa157d7ad203ac108c58d34df7fae789 +Author: Jacques B +Date: Sun Dec 9 15:03:48 2018 +0100 + + RestClient: on retry timeout add root exception (#25576) + +commit 95bafb0593360dd32f2d0dedcbd947ce42dc48df +Author: Nhat Nguyen +Date: Sun Dec 9 02:57:13 2018 -0500 + + TEST: Always enable soft-deletes in ShardChangesTests + +commit 32c4f99238112fa4b381316c1e73d457956ef040 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sun Dec 9 16:03:28 2018 +1100 + + [HLRC] Add support for put privileges API (#35679) + + This commit adds support for API to create or update + application privileges in high-level rest client. + +commit 902d6f579aff968d8f8426f1e6855ca1d9fb3a05 +Author: Nhat Nguyen +Date: Sat Dec 8 21:27:44 2018 -0500 + + Exposed engine must have all ops below gcp during rollback (#36159) + + Today we expose a new engine immediately during Lucene rollback. The new + engine is started with a safe commit which might not include all + acknowledged operation. With this change, we won't expose the new engine + until it has recovered from the local translog. + + Note that this solution is not complete since it's able to reserve only + acknowledged operations before the global checkpoint. This is because we + replay translog up to the global checkpoint during rollback. A per-doc + Lucene rollback would solve this issue entirely. + + Relates #32867 + +commit d41cf6ac9f791b431762892b4ce8e85419e44eba +Author: Nhat Nguyen +Date: Sat Dec 8 21:24:36 2018 -0500 + + Use delCount of SegmentInfos to calculate numDocs (#36323) + + Today, we iterate the bitset of hardLiveDocs to calculate the number of + live docs. This calculation might be expensive if we enable soft-deletes + (by default) for old indices whose soft-deletes was disabled previously + and had hard-deletes. + + Once soft-deletes is enabled, we no longer hard-update or hard-delete + documents directly. We have hard-deletes in two scenarios: (1) from old + segments where soft-deletes was disabled, (2) when IndexWriter hits + non-aborted exceptions. These two cases, IW flushes SegmentInfos before + exposing the hard-deletes; thus we can use the hard-delete count of + SegmentInfos. + +commit cc3872d934954d07d15ca8d7b8b1d48cc8c560de +Author: Tim Brooks +Date: Sat Dec 8 14:24:38 2018 -0700 + + Reduce composite references at network level (#36400) + + Creating `CompositeBytesReference` has more overhead than a single + `ByteBufferReference`. Many of our messages will be contained to a + single `ByteBuffer`. This commit avoids creating composite instances + when there is 0 or 1 underlying `ByteBuffers`. + +commit b5b6e37a60fcd5ee0ecf707c19eac5b919750e6b +Author: Michael Basnight +Date: Sat Dec 8 12:57:16 2018 -0600 + + Deprecate X-Pack centric watcher endpoints (#36218) + + This commit is part of our plan to deprecate and ultimately remove the use of + _xpack in the REST APIs. + + Relates #35958 + +commit 9f86e996fecd8a6e6aa4a2b3c5e2580e1c3171db +Author: David Turner +Date: Sat Dec 8 07:33:35 2018 +0000 + + [Zen2] Support rolling upgrades from Zen1 (#35737) + + We support rolling upgrades from Zen1 by keeping the master as a Zen1 node + until there are no more Zen1 nodes in the cluster, using the following + principles: + + - Zen1 nodes will never vote for Zen2 nodes + - Zen2 nodes will, while not bootstrapped, vote for Zen1 nodes + - Zen2 nodes that were previously part of a mixed cluster will automatically + (and unsafely) bootstrap themselves when the last Zen1 node leaves. + +commit a27f2efca555da798de961722c7b7ad61c3d0288 +Author: Ryan Ernst +Date: Fri Dec 7 17:23:41 2018 -0800 + + Core: Converge FormatDateTimeFormatter and DateFormatter apis (#36390) + + This commit makes FormatDateTimeFormatter and DateFormatter apis close + to each other, so that the former can be removed in favor of the latter. + This PR does not change the uses of FormatDateTimeFormatter yet, so that + that future change can be purely mechanical. + +commit b15d1aebcf742f1d95660f7005f672a57bfaccd0 +Author: Julie Tibshirani +Date: Fri Dec 7 14:50:47 2018 -0800 + + For msearch templates, make sure to use the right name for deprecation logging. (#36344) + +commit 85bba0c3ae031fd19027159aac3aba960546690c +Author: Gordon Brown +Date: Fri Dec 7 15:45:53 2018 -0700 + + Simplify deprecation issue levels (#36326) + + This commit gets rid of the 'NONE' and 'INFO' severity levels for + deprecation issues. + + 'NONE' is unused and does not make much sense as a severity level. + 'INFO' can be separated into two categories: Either 1) we can + definitively tell there will be a problem with the cluster/node/index + configuration that can be resolved prior to upgrade, in which case + the issue should be a WARNING, or 2) we can't, because any issues would + be at the application level, for which the user should review the + deprecation logs and/or response headers. + +commit 2ba023be8a57b5dcd8dad172ea26a8e0d0e01717 +Author: Simon Willnauer +Date: Fri Dec 7 23:38:35 2018 +0100 + + Explicitly recommend to forceMerge before freezing (#36376) + + given the benchmark results on #34352 it's important to recommend + users to `_force_merge` their indices to a single segment before freezing. + +commit 8a53f2b46489e3520fc333c13d7c71d830c8d34b +Author: Tim Brooks +Date: Fri Dec 7 15:27:04 2018 -0700 + + Implement basic `CcrRepository` restore (#36287) + + This is related to #35975. It implements a basic restore functionality + for the CcrRepository. When the restore process is kicked off, it + configures the new index as expected for a follower index. This means + that the index has a different uuid, the version is not incremented, and + the Ccr metadata is installed. + + When the restore shard method is called, an empty shard is initialized. + +commit a998f4dec60a5f2df76d2573b11828b97386ef14 +Author: Ryan Ernst +Date: Fri Dec 7 12:40:14 2018 -0800 + + Core: Remove parseDefaulting from DateFormatter (#36386) + + This commit removes the parseDefaulting method from DateFormatter, + bringing it more inline with the joda equivalent + FormatDateTimeFormatter. This method was only needed for the java + time implementation of DateMathParser. Instead, a DateFormatter now + returns an implementation of DateMathParser for the given format, + allowing the java time implementation to construct the appropriate date + math parser internally. + +commit 9e8cfbb40d7596eb56151fc8a30234fb336c0a43 +Author: David Roberts +Date: Fri Dec 7 20:34:11 2018 +0000 + + [ML] Deprecate X-Pack centric ML endpoints (#36315) + + This commit is part of our plan to deprecate and + ultimately remove the use of _xpack in the REST APIs. + + Relates #35958 + +commit 1d73c1712bbf4bc9af3c441570c94541a763e591 +Author: Wilder Pereira +Date: Fri Dec 7 18:18:22 2018 -0200 + + Docs: Fix Jackson reference (#36366) + + Link is totally broken. + +commit 7870ae81b151ca6bc189551362eae2a860e83253 +Author: Dimitris Athanasiou +Date: Fri Dec 7 20:16:25 2018 +0000 + + [ML] Prevent stack overflow while copying ML jobs and datafeeds (#36370) + + ML jobs and datafeeds wrap collections into their unmodifiable + equivalents in their constructor. However, the copying builder + does not make a copy of some of those collections resulting + in wrapping those again and again. This can eventually result + to stack overflow. + + This commit addressed this issue by copying the collections in + question in the copying builder constructor. + + Closes #36360 + +commit ead2b9e08b0c4ec63f3325e1aba2d2cfa70ef48b +Author: Nik Everett +Date: Fri Dec 7 14:39:58 2018 -0500 + + HLRC: Add rollup search (#36334) + + Relates to #29827 + +commit 51e1d40dca00385d9b8946fd38c000f226a8830f +Author: Julie Tibshirani +Date: Fri Dec 7 11:21:24 2018 -0800 + + Small improvements related to types deprecation. (#36328) + + * Make sure to use deprecatedAndMaybeLog for types deprecation messages. + * Introduce a common base class for Rest*Action tests. + +commit 5556204f813e973e90c08f06b36da5aab91075c1 +Author: Tim Brooks +Date: Fri Dec 7 11:17:11 2018 -0700 + + Use MockNioTransport in MockTransportService (#36346) + + The default transport used in the MockTransportService is the + MockTcpTransport. This commit changes that to be the + MockNioTransport. + +commit 03c577b477273d44e502ccc5350d8bf72716d6e6 +Author: Alpar Torok +Date: Fri Dec 7 20:07:36 2018 +0200 + + Move VersionProperties and Version to min runtime (#36378) + + Allow elasticsearch-hadoop to access these classes when running on + java 8. + +commit b8dba16376c29ca31bc6907a961949d3903164da +Author: Dimitris Athanasiou +Date: Fri Dec 7 18:01:37 2018 +0000 + + [ML] Ensure total hits are tracked (#36374) + + This is in preparation of the anticipated change + that will disable accurate total hits tracking in + searches. + +commit 2df4bd1f8135fca889312711979c13a61b90fafd +Author: Jack Conradson +Date: Fri Dec 7 09:32:27 2018 -0800 + + [Painless] Generate Bridge Methods (#36097) + + We use MethodHandles.asType to cast argument types into the appropriate parameter types for + method calls when the target of the call is a def type at runtime. Currently, certain implicit casts + using the def type are asymmetric. It is possible to cast Integer -> float as an argument to parameter, but not from int -> Float (boxed to primitive with upcasting is okay, but primitive to + boxed with upcasting is not). + + This PR introduces a solution to the issue by generating bridge methods for all whitelisted methods + that have at least a single boxed type as an argument. The bridge method will conduct appropriate + casts and then call the original method. This adds a bit of overhead for correctness. It should not be + used often as Painless avoids boxed types as much as possible. + + Note that a large portion of this change is adding methods to do the appropriate def to boxed type + casts and a few mechanical changes as well. The most important method for review is + generateBridgeMethod in PainlessLookupBuilder. + +commit 9d417984bde50608f4c58b76dfbed64e3e2ff17e +Author: David Turner +Date: Fri Dec 7 17:23:20 2018 +0000 + + [Zen2] Add warning if cluster fails to form fast enough (#35993) + + * Add warning if cluster fails to form fast enough + + Today if a leader is not discovered or elected then nodes are essentially + silent at INFO and above, and log copiously at DEBUG and below. A short delay + when electing a leader is not unusual, for instance if other nodes have not yet + started, but a persistent failure to elect a leader is a problem worthy of log + messages in the default configuration. + + With this change, while there is no leader each node outputs a WARN-level log + message every 10 seconds (by default) indicating as such, describing the + current discovery state and the current quorum(s). + + * Add note about whether the discovered nodes form a quorum or not + + * Introduce separate ClusterFormationFailureHelper + + ... and back out the unnecessary changes elsewhere + + * It can be volatile + +commit f2df0a5be4da208edb2606ef8e522558f2f5f921 +Author: Nhat Nguyen +Date: Fri Dec 7 12:22:20 2018 -0500 + + Remove LocalCheckpointTracker#resetCheckpoint (#34667) + + In #34474, we added a new assertion to ensure that the + LocalCheckpointTracker is always consistent with Lucene index. However, + we reset LocalCheckpoinTracker in testDedupByPrimaryTerm cause this + assertion to be violated. + + This commit removes resetCheckpoint from LocalCheckpointTracker and + rewrites testDedupByPrimaryTerm without resetting the local checkpoint. + + Relates #34474 + +commit cec788deead9e932efdce835059f644c3df5edc2 +Author: Nhat Nguyen +Date: Fri Dec 7 12:14:42 2018 -0500 + + HLRC/TEST: Fix document IT for delete template API + + Relates #36320 + +commit 32a7d7abd56634c8967a316522f9c8bc2c517be1 +Author: Nhat Nguyen +Date: Fri Dec 7 10:46:03 2018 -0500 + + HLRC: Add delete template API (#36320) + + Relates #27205 + +commit 844ff807789974bc733692af88a69d49bde51b99 +Author: Simon Willnauer +Date: Fri Dec 7 15:38:16 2018 +0100 + + Harden FrozenEngineTests#testCircuitBreakerAccounting + + This test tries to compare the CB stats from an InternalEngine + and a FrozenEngine but is subject to segement merges that might finish + and get committed after we read the breaker stats. This can cause + occational test failures. + + Closes #36207 + +commit a53e8653f292b9a4ff37bdee1f9d535b82c546e7 +Author: Jim Ferenczi +Date: Fri Dec 7 15:26:08 2018 +0100 + + Add support for inlined user dictionary in Nori (#36123) + + Add support for inlined user dictionary in Nori + + This change adds a new option called `user_dictionary_rules` to the + Nori a tokenizer`. It can be used to set additional tokenization rules + to the Korean tokenizer directly in the settings (instead of using a file). + + Closes #35842 + +commit ca09936cdf31c40dcf09233638a3d1d9d3fee362 +Author: Michael Basnight +Date: Fri Dec 7 08:22:21 2018 -0600 + + HLRC: Get Deprecation Info API (#36279) + + This commit adds the Get Deprecation Info API and associated + documentation. + + Relates #29827 + +commit c32e4fb83f78e350cc44ad1c616a0af1577de93c +Author: David Turner +Date: Fri Dec 7 12:47:09 2018 +0000 + + [Zen2] Best-effort cluster formation if unconfigured (#36215) + + In real deployments it is important that clusters are properly configured to + avoid accidentally forming multiple independent clusters at cluster + bootstrapping time. However we also expect to be able to unpack Elasticsearch + and start up one or more nodes without any up-front configuration, and have + them do their best to find each other and form a cluster after a few seconds. + + This change adds a delayed automatic bootstrapping process to nodes that start + up with no relevant settings set to support the desired out-of-the-box + experience without compromising safety in properly-configured deployments. + +commit 879397df703bf2077ab6351c776240aad2574a4b +Author: Christoph Büscher +Date: Fri Dec 7 11:15:16 2018 +0100 + + Remove deprecated methods in SimpleQueryStringBuilder (#36341) + + The `useAllFields`methods have already been deprecated in 6.0 and are save to remove in 7.0. + +commit 562479b5a57838cae1e8c8eb2f0306d8e42ec044 +Author: Christoph Büscher +Date: Fri Dec 7 11:14:41 2018 +0100 + + Remove deprecated methods in MoreLikeThisQueryBuilder (#36340) + + The ids method is deprecated since 6.0 (at least) and only used in a few tests + internally, so we can safely remove it. + +commit 60289e7331951c7dcf475ec00deb0c147e0465be +Author: David Roberts +Date: Fri Dec 7 09:49:55 2018 +0000 + + Mute SourceOnlySnapshotIT testSnapshotAndRestore + + Due to https://github.com/elastic/elasticsearch/issues/36330 + +commit 0dd73ef7daa8c63d8d7e8dd7ccfbeff6aa45bdb4 +Author: Dimitris Athanasiou +Date: Fri Dec 7 09:33:51 2018 +0000 + + [ML] Move consuming and closing results stream to result processor (#36314) + + The results iterator is consuming and closing the results stream + once it is done. It seems this should not be the responsibility + of the results iterator. It stops the iterator from being reusable + for different processes where closing the stream is not desirable. + This commit is moving the consuming and closing of the results stream + into the autodetect result processor. + +commit 266b9bc796be9944c2aaf670c3b5eb467dfff454 +Author: Adrien Grand +Date: Fri Dec 7 09:57:12 2018 +0100 + + Inner hits fail to propagate doc-value format. (#36310) + + If you pass a doc-value format to inner hits, it gets ignored. + +commit 6a987415f8738d8d0e272390edeef083946ce2ec +Author: Alexander Reelsen +Date: Fri Dec 7 09:40:58 2018 +0100 + + Scripting: Properly support no-offset date formatting (#36316) + + The conversion of timezones in JodaCompatibleZonedDateTime from joda to + java time requires the use of the DateUtils class to cater for corner + cases. + + Closes #36306 + +commit 7693d538ca22ec439c8e069f785c2c0a1ffc0226 +Author: Alexander Reelsen +Date: Fri Dec 7 09:38:12 2018 +0100 + + Docs: Mention that fields with dots do not work (#36295) + + The dot is used as a splitting character internally for looking up + values in the array compare condition, thus the user should use the + script condition in such cases. + +commit 593d8637c4879f4919372d7e1aeffbf17da24157 +Merge: 3ca885e9ac6 9691d0508c7 +Author: Tanguy Leroux +Date: Fri Dec 7 09:31:35 2018 +0100 + + Merge branch 'master' into close-index-api-refactoring + +commit 3ca885e9ac6b031b896e62bece07170abe0ac85c +Author: Tanguy Leroux +Date: Fri Dec 7 09:30:27 2018 +0100 + + [Close Index API] Add TransportShardCloseAction for pre-closing verifications (#36249) + + This pull request adds the TransportShardCloseAction which is a + transport replication action that acquires all index shard permits for + its execution. This action will be used in the future by the + MetaDataIndexStateService in a new index closing process, where + we need to execute some sanity checks before closing an index. + + The action executes the following verifications on the primary and replicas: + * there is no other on going operation active on the shard + * the data node holding the shard knows that the index is blocked for writes + * the shard's max sequence number is equal to the global checkpoint + + When the verifications are done and successful, the shard is flushed. + + Relates #33888 + +commit 9691d0508c7a90749ab5b97cb5807788d7d220c6 +Author: Tanguy Leroux +Date: Fri Dec 7 09:20:24 2018 +0100 + + Remove unused class DelegatingActionListener (#36257) + +commit e6d190613f36504a724be46b5e10659066f29a79 +Author: Armin Braun +Date: Fri Dec 7 09:15:11 2018 +0100 + + [ZEN2] Use Zen2 in REST Tests (#36300) + +commit 968b0b1992d26effdda576a4c9fc2b9da98a197c +Author: Nhat Nguyen +Date: Fri Dec 7 03:01:04 2018 -0500 + + Add soft-deletes upgrade tests (#36286) + + This change adds a rolling-upgrade and full-cluster-restart test with + soft-deletes enabled. + +commit 10feb75eb77ad64aec9e1f0a7ec824338fb4c7de +Author: Nhat Nguyen +Date: Thu Dec 6 20:33:25 2018 -0500 + + Upgrade to Lucene-8.0.0-snapshot-aaa64d70159 (#36335) + + Includes: + + LUCENE-8594: DV update are broken for updates on new field + LUCENE-8590: Optimize DocValues update datastructures + LUCENE-8593: Specialize single value numeric DV updates + + Relates #36286 + +commit 373c67dd7ab31d5b57a2efa32b4188c9dc61449d +Author: Tim Brooks +Date: Thu Dec 6 18:09:07 2018 -0700 + + Add DirectByteBuffer strategy for transport-nio (#36289) + + This is related to #27260. In Elasticsearch all of the messages that we + serialize to write to the network are composed of heap bytes. When you + read or write to a nio socket in java, the heap memory you passed down + must be copied to/from direct memory. The JVM internally does some + buffering of the direct memory, however it is essentially unbounded. + + This commit introduces a simple mechanism of buffering and copying the + memory in transport-nio. Each network event loop is given a 64kb + DirectByteBuffer. When we go to read we use this buffer and copy the + data after the read. Additionally, when we go to write, we copy the data + to the direct memory before calling write. 64KB is chosen as this is the + default receive buffer size we use for transport-netty4 + (NETTY_RECEIVE_PREDICTOR_SIZE). + + Since we only have one buffer per thread, we could afford larger. + However, if we the buffer is large and not all of the data is flushed in + a write call, we will do excess copies. This is something we can + explore in the future. + +commit fc85c37efc02be9a0eca70654eac5d88359fc78d +Author: Jason Tedor +Date: Thu Dec 6 19:20:10 2018 -0500 + + Fix typo in migration node for one shard per index + +commit e8fe624570b1dc815e82f2057e0f6127f78cc557 +Author: Jason Tedor +Date: Thu Dec 6 19:15:45 2018 -0500 + + Add migration note on the number of shards + + This commit adds a migration note regarding the default number of shards + changing from five to one. + + Relates #30539 + +commit a7edf1ed1ff5ba72bae39e053e955576ce5b28ad +Author: Nhat Nguyen +Date: Thu Dec 6 18:37:54 2018 -0500 + + TEST: Ensure a manual flush always occur (#36188) + + The test testLookupSeqNoByIdInLucene fails because it expects if any + change should be visible after a flush. However, that flush might be + ignored if the waitIfOngoing parameter is false (the default value), and + there is an ongoing flush triggered after merge is running. + + Closes #35823 + +commit 0b7b78b87498afdcb1e66395229fa4c6007bda22 +Author: Jack Conradson +Date: Thu Dec 6 14:49:43 2018 -0800 + + Mute test CCRIT testIndexFollowing (#36339) + +commit 37b3fc383fd4fd9e78b4b218b3a664c6c259e4fd +Author: Ryan Ernst +Date: Thu Dec 6 14:13:49 2018 -0800 + + Build: Use explicit deps on test tasks for check (#36325) + + This commit moves back to use explicit dependsOn for test tasks on + check. Not all tasks extending RandomizedTestingTask should be run by + check directly. + +commit ed45acf221c1021b116c3b0f2cbeba79d00401e3 +Author: Jason Tedor +Date: Thu Dec 6 17:10:45 2018 -0500 + + Fix name of Docker binaries + + This commit fixes an oops when pushing a change to add the building of + the Docker images. A change that was made for testing was accidentally + left behind. This commit addresses that. + +commit 11dd412ec6ccfab9b99201ebd85f80e06eefd7ad +Author: Jason Tedor +Date: Thu Dec 6 17:06:13 2018 -0500 + + Introduce Docker images build (#36246) + + This commit introduces the building of the Docker images as bonafide + packaging formats alongside our existing archive and packaging + distributions. This build is migrated from a dedicated repository, and + converted to Gradle in the process. + +commit 3e04a90e9924f333f018c74a4c981eace7cb562b +Author: Benjamin Trent +Date: Thu Dec 6 15:58:37 2018 -0600 + + [ML] Adding audits when deprecation warnings occur with datafeed start (#36233) + + * [ML] Adding audits when deprecation warnings occur with datafeed start + + * adjusting parameters for log format call + +commit d1184cfb6849d9e4b6c13e544360ffcb1cb5b06d +Author: Jason Tedor +Date: Thu Dec 6 16:07:40 2018 -0500 + + Add docs on replicating APM Server or Beats indices (#36333) + + This commit adds a brief note to the documentation on how to manage the + index templates that are used to create APM Server and Beats indices. + +commit e944764be31c6122dfb7df9a2f63d1669625a2da +Author: Tal Levy +Date: Thu Dec 6 10:41:55 2018 -0800 + + [ILM] [DOCS] add general info about steps (#36081) + + it is important for users to understand that phases + are compiled into Steps that are exposed via the + Explain Lifecycle API. + +commit 3f3cde41d39b20b5b4408716db89bd059ef77561 +Author: Julie Tibshirani +Date: Thu Dec 6 10:23:15 2018 -0800 + + Deprecate types in termvector and mtermvector requests. (#36182) + + * Add deprecation warnings to `Rest*TermVectorsAction`, plus tests in `Rest*TermVectorsActionTests`. + * Deprecate relevant methods on the Java HLRC requests/ responses. + * Update documentation (for both the REST API and Java HLRC). + * For each REST yml test, create one version without types, and another legacy version that retains types (called *_with_types.yml). + +commit fb697c74421ab60622e9b5cd56cfab797167cc36 +Author: Lisa Cawley +Date: Thu Dec 6 10:14:37 2018 -0800 + + [DOCS] Add missing anchors (#36288) + +commit d4d3a3e46750b0a9ed078864bb6111882946b687 +Author: Jason Tedor +Date: Thu Dec 6 12:52:04 2018 -0500 + + Remove license state listeners on closables (#36308) + + We have a few places where we register license state listeners on + transient components (i.e., resources that can be open and closed during + the lifecycle of the server). In one case (the opt-out query cache) we + were never removing the registered listener, effectively a terrible + memory leak. In another case, we were not un-registered the listener + that we registered, since we were not referencing the same instance of + Runnable. This commit does two things: + - introduces a marker interface LicenseStateListener so that it is + easier to identify these listeners in the codebase and avoid classes + that need to register a license state listener from having to + implement Runnable which carries a different semantic meaning than + we want here + - fixes the two places where we are currently leaking license state + listeners + +commit adc8355c5d905b89fbe8215ff93c024f49fa7536 +Author: Benjamin Trent +Date: Thu Dec 6 11:19:34 2018 -0600 + + Adding additional tests for agg parsing in datafeedconfig (#36261) + + * Adding additional tests for agg parsing in datafeedconfig + + * Fixing bug, adding yml test + +commit 96ee6e0d2afe1c65a91f9f288ed2b53647c7beee +Author: Armin Braun +Date: Thu Dec 6 17:55:40 2018 +0100 + + TEST: Reenable RemoveCorruptedShardDataCommandIT.testCorruptIndex + + * Merging the zen2 branch has made it impossible to reproduce the test failure here, likely as a result of the state now being written atomically + * Closes #36189 + +commit 65e15f07fbebb2b3596f1bc97aeb2439d73d4fc3 +Author: Christoph Büscher +Date: Thu Dec 6 17:40:59 2018 +0100 + + Add comments about need for explicit cast + + Some IDEs (specifically Eclipse 4.8.0) needs some explicit casting for type + inference. This was added in a recent change but we should also have comments so + we remember why this needs to be there (at least for now). + +commit ed1c5a024168551764687539689d033cd54483ff +Author: David Turner +Date: Thu Dec 6 16:20:08 2018 +0000 + + Introduce `zen2` discovery type (#36298) + + With this change it is now possible to start a node running Zen2. + +commit f0340d6d32d58155273a3135762490329e720ae5 +Author: Andrey Ershov +Date: Thu Dec 6 17:02:02 2018 +0100 + + [Zen2] Storage layer WriteStateException propagation (#36052) + + Currently, we only log that WriteStateException has occurred, in + GatewayMetaState. + + This PR goal is to re-throw WriteStateException to upper layers. + If dirty flag is set to false, we wrap WriteStateException in + UncheckedIOException, we prefer unchecked exception not to add + explicit throws in the multitude of layers. + If dirty flag is set to true - the world is broken. And we need to + halt the JVM. Instead of explicit halting in GatewayMetaState, we + prefer to throw IOError, which will be subsequently handled by + ElasticsearchUncaughtExceptionHandler and JVM will be halted. + + This PR also adds tests for WriteStateException. + +commit b08cffad48ed3833e69611fb5bfbc9bb85c2f72b +Author: Adrien Grand +Date: Thu Dec 6 16:31:12 2018 +0100 + + Fix line length offenders in the o.e.search package. (#36223) + + Relates #34884 + +commit 6186ccf83e78cd267cbb33be230e56d3a46613f9 +Author: Tanguy Leroux +Date: Thu Dec 6 16:05:05 2018 +0100 + + [Tests] Fix third party tests with Gradle 5.0 (#36302) + + * [Tests] Fix third party tests with Gradle 5.0 + + * apply feedback + +commit f5fa1f4746fe66a09a90a264728c4c13f65a993e +Author: Christoph Büscher +Date: Thu Dec 6 16:04:25 2018 +0100 + + Help Eclipse with type inference for functions (#36301) + + The Eclipse IDE java compiler seems to need some special hints about what types + some functions used in the tests return. Correcting this for some test that were + newly merged to master. + +commit 97259f0efc4f808b33da34509b2a8a5f8d6ef9cf +Author: Alexander Reelsen +Date: Thu Dec 6 14:35:53 2018 +0100 + + Docs: Add password keystore setting for email account passwords (#33409) + + Always refer to secure settings when setting up email account passwords + +commit 98e66c6afe9ea04737c9f3544b95c43c8087461e +Author: Armin Braun +Date: Thu Dec 6 13:43:28 2018 +0100 + + SNAPSHOT: Increase Timeout to Stabilize Test (#36294) + + * 30s is pretty close here, most runs take 25s of the timeout making the test unstable + * closes #36283 + +commit 77e6ef7b20720fd859beb2f5ebc79ee80b311b75 +Author: Ioannis Kakavas +Date: Thu Dec 6 12:44:51 2018 +0200 + + Fix get certificates HLRC API (#36198) + + - GetSslCertificatesRequest need not implement toXContentObject + - getRequest() returns a new Request object + - Add tests for GetSslCertificatesResponse + - Adjust docs to the new format + +commit ee05ef131286eead295cf58ed4393af56f0d7ade +Merge: 9ae5ee24974 38ab15c6fb6 +Author: Yannick Welsch +Date: Thu Dec 6 08:31:46 2018 +0100 + + Merge branch 'zen2' + +commit 38ab15c6fb6eb4fe15e739faedb19858344ba436 +Author: David Turner +Date: Thu Dec 6 07:27:38 2018 +0000 + + Avoid shutting down the only master (#36272) + + Today the InternalTestClusterTests sometimes set up a cluster with a single + master, start some other ndoes, shut the original master down, and then reset + the cluster. This doesn't really work, because the original master may be + stale. This change avoids shutting down the only master in this situation. + +commit 9ae5ee24974b251b71f15510bc15284a8c6de2b6 +Author: Alpar Torok +Date: Thu Dec 6 09:14:50 2018 +0200 + + Fix typo in comment + +commit 3ac64c1ec2078f2f50ad00328eb7cdad341a52ca +Author: Jim Ferenczi +Date: Thu Dec 6 07:02:22 2018 +0100 + + Fix total hits serialization of the search response (#36290) + + This change removes the custom serialization of the total hits + and reuses the shard's serialization of Lucene#read/writeTopDocs + in the client code. This also removes the incorrect assertion that + trips randomly in bwc tests. + + Closes #36284 + +commit 3652f9055246bce4907f1dee740a5dd0994566df +Author: Jim Ferenczi +Date: Thu Dec 6 02:45:07 2018 +0100 + + Fix FullClusterRestartIT#testRollupIDSchemeAfterRestart + + Set rest_total_hits_as_int in search requests only on the new cluster, + the old cluster can be on a version older than don't support this + parameter (< 6.6) and total hits is not an object in 6x anyway. + + Closes #36291 + +commit 331026b59776e1289efb6eac817eec9a160b0de1 +Author: Julie Tibshirani +Date: Wed Dec 5 17:05:28 2018 -0800 + + Mute FullClusterRestartIT#testRollupIDSchemeAfterRestart as we await a fix. + +commit 54f39d9852abe0159af131c53357ebd71f664ccb +Author: Christoph Büscher +Date: Thu Dec 6 00:09:26 2018 +0100 + + [Docs] Add Profile API limitations (#36252) + + Adding some of the limitations mentioned in #29275. + + Closes #29275 + +commit 75635251ab7eb5db1869b13a049f46a11a9b13cd +Author: Christoph Büscher +Date: Thu Dec 6 00:07:15 2018 +0100 + + Make sure test don't use Math.random for reproducability (#36241) + + Currently we use Math.random() in a few places in the tests which makes these + tests not reproducable with the random seed mechanism that comes with + ESTestCase. The change removes those instances. + +commit 38d2c093ad8e6b4e9daf8759c75dd6615a7373b7 +Author: Yannick Welsch +Date: Wed Dec 5 23:44:33 2018 +0100 + + Fix compilation + +commit 190ac8e9bfcface46e3061b6f6ba3a238a7ce1f3 +Author: Jake Landis +Date: Wed Dec 5 16:25:50 2018 -0600 + + ingest: support default pipeline through an alias (#36231) + + This commit allows writes that go through an alias to use the default + pipeline defined on the backing index. + + Fixes #35817 + +commit a0ae1cc98728d95a39ae5f166b1642b767ab797c +Merge: 03d0ea91ef5 2d03eeb632d +Author: Yannick Welsch +Date: Wed Dec 5 23:13:12 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 03d0ea91ef5205d41aa14506ccc46609d3716622 +Author: Yannick Welsch +Date: Wed Dec 5 23:12:28 2018 +0100 + + Zen2: Rename tombstones to exclusions (#36226) + + Renames the withdrawal / tombstones APIs to voting configuration exclusions. + +commit 5d6602120fd6c5368954d3cb501a418ebc083d8a +Author: Andrey Ershov +Date: Wed Dec 5 23:11:20 2018 +0100 + + [Zen2] Hide not recovered state (#36224) + + This commit hides ClusterStates that have a STATE_NOT_RECOVERED_BLOCK from + ClusterStateAppliers. This is needed, because some appliers, such as IngestService, rely on + the fact, that cluster states with STATE_NOT_RECOVERED_BLOCK won't contain anything useful. + Once the state is recovered it's fully available for the appliers. This commit also switches many of + the remaining tests that require state persistence/recovery from Zen1 to Zen2. + +commit 2d03eeb632d098f66ae0cefdbb5d28150a4ea942 +Author: jaymode +Date: Wed Dec 5 14:06:38 2018 -0700 + + Test: mute testDataNodeRestartWithBusyMasterDuringSnapshot + + This commit mutes DedicatedClusterSnapshotRestoreIT# + testDataNodeRestartWithBusyMasterDuringSnapshot which fails reproducibly. + + See #36283 + +commit b8743f6636696f12ce229053c23d7d95261d1fb8 +Author: jaymode +Date: Wed Dec 5 13:44:11 2018 -0700 + + Test: mute testSnapshotAndRestoreWithNested + + This mutes SourceOnlySnapshotIT#testSnapshotAndRestoreWithNested as it + fails reproducibly. + + See #36276 + +commit 54dcc273a801d4ad48f7e2ae5c73aec7836fc4cb +Author: jaymode +Date: Wed Dec 5 13:38:37 2018 -0700 + + Revert "Test: mute failing mtermvector rest test" + + This reverts commit 5ba0ed9c8f57b954a122a5b364b09da5551118fd as this + test was already fixed by 4c2e0d76f32090c2731a1fa8a8a6b3ede828ccb5. + + Closes #36281 + +commit 5ba0ed9c8f57b954a122a5b364b09da5551118fd +Author: jaymode +Date: Wed Dec 5 12:52:46 2018 -0700 + + Test: mute failing mtermvector rest test + + See #36281 + +commit dba3f4ec9f4664e7a0cccdac668d9579e47d04eb +Author: Jay Modi +Date: Wed Dec 5 12:15:09 2018 -0700 + + add version 6.5.3 (#36268) + +commit 18866c4c0b1ad640f8554dc4b41765a04dcefa04 +Author: Jim Ferenczi +Date: Wed Dec 5 19:49:06 2018 +0100 + + Make hits.total an object in the search response (#35849) + + This commit changes the format of the `hits.total` in the search response to be an object with + a `value` and a `relation`. The `value` indicates the number of hits that match the query and the + `relation` indicates whether the number is accurate (in which case the relation is equals to `eq`) + or a lower bound of the total (in which case it is equals to `gte`). + This change also adds a parameter called `rest_total_hits_as_int` that can be used in the + search APIs to opt out from this change (retrieve the total hits as a number in the rest response). + Note that currently all search responses are accurate (`track_total_hits: true`) or they don't contain + `hits.total` (`track_total_hits: true`). We'll add a way to get a lower bound of the total hits in a + follow up (to allow numbers to be passed to `track_total_hits`). + + Relates #33028 + +commit c4181dc569aaef1ebbc5aab2e106760668a30203 +Author: lcawl +Date: Wed Dec 5 09:53:21 2018 -0800 + + [DOCS] Fixes broken link in execute watch + +commit 26e271c285d3e1a3abe181f305d2eb2efba38f84 +Author: Alan Woodward +Date: Wed Dec 5 16:57:38 2018 +0000 + + Correct doc reference tag (#36262) + + Fixes a broken reference in the watcher documentation. + +commit ce7b1886b8f63ea1564aed75c8de409130740634 +Author: Mark Tozzi +Date: Wed Dec 5 10:56:48 2018 -0500 + + Unit test for very large percentile aggs (#36122) + + Closes #19528 + + * Port a test Colin wrote for the TDigest library to validate TDigests storing over 2*MAXINT values. This appears to have been fixed in version 3.2 of TDigest, which Elasticsearch has been using for some time, so no changes were necessary to resolve this issue. + +commit cc1195372420e50ae810567ea86d0f6dd0d7a997 +Merge: e8289167ee7 068c856e884 +Author: Yannick Welsch +Date: Wed Dec 5 16:55:45 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 068c856e884b59ab9630a428845fe04d5e97c7e3 +Author: Tim Brooks +Date: Wed Dec 5 08:11:47 2018 -0700 + + Rename internal repository actions to be internal (#36244) + + This is a follow-up to #36086. It renames the internal repository + actions to be prefixed by "internal". This allows the system user to + execute the actions. + + Additionally, this PR stops casting Client to NodeClient. The client we + have is a NodeClient so executing the actions will be local. + +commit a3c1c6938a374dda7de0ad48ae35d3842c6805fe +Author: David Roberts +Date: Wed Dec 5 14:41:07 2018 +0000 + + Mute ForecastIT.testSingleSeries + + Due to https://github.com/elastic/elasticsearch/issues/36258 + +commit e8289167ee75122d22aa0c8d6aeaf0a8332a964c +Author: Andrey Ershov +Date: Wed Dec 5 14:59:41 2018 +0100 + + Fix FAIL_ON_METHOD in MetaDataStateFormatTests + + fail method should be computed before eval, not inside it + +commit 3d85e8cd0dab332b1635020cfe3b1079278ee64b +Author: Martijn van Groningen +Date: Wed Dec 5 14:12:38 2018 +0100 + + [TEST] Use different name for remote cluster connection + + To avoid collisions with other tests. + +commit b20497560cc6ba16ba18d94a5affdbf103cb9832 +Merge: 0b9efff5cbd 37127c19ba1 +Author: Yannick Welsch +Date: Wed Dec 5 14:06:38 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 46209591fe250ffd002325b3110bbe8b34006b50 +Author: Adrien Grand +Date: Wed Dec 5 14:06:32 2018 +0100 + + Propagate a doc-value `format` with inner hits. (#36068) + + While sql passes a doc-value `format` when asking for doc-value fields, it + doesn't do it when asking for fields via inner hits. + +commit 0b9efff5cbdbab94d4c8b72fc5f2c1c10ca69ba7 +Author: Yannick Welsch +Date: Wed Dec 5 14:04:45 2018 +0100 + + Zen2: Persist cluster states the old way on non-master-eligible nodes (#36247) + + The shard deletion logic (triggered by IndicesStore), which also leads to index metadata deletion on + non-master-eligible data nodes, currently races against the new cluster state persistence logic + triggered by accepting cluster states. One thread is writing the index metadata while another one is + deleting the index metadata, leading to exceptions and assertions tripping (see below). The solution + proposed by this PR is to move the cluster state persistence of non-master-eligible nodes back to + the cluster applier service, just as it used to be for Zen1. This ensures that the index metadata + deletion logic, which is triggered by the shard deletion logic, runs on the same thread on which we + persist the cluster state. + +commit 37127c19ba19ad97fe35ccb291d4fefcd9ecc0f6 +Author: Gil Raphaelli +Date: Wed Dec 5 07:46:17 2018 -0500 + + add missing error type mapping for apm-server monitoring (#36178) + +commit d531fab4b30411cd24848c50fbe35c225ccddb6b +Author: Alpar Torok +Date: Wed Dec 5 14:44:05 2018 +0200 + + Run Build tests with proper java version + +commit 73ceaad03a62b99538ef244cb2ee83b1eef6db8a +Author: Alan Woodward +Date: Wed Dec 5 12:43:56 2018 +0000 + + Update to lucene-8.0.0-snapshot-c78429a554 (#36212) + + Includes: + + * A fix for a bug in Intervals.or() (https://issues.apache.org/jira/browse/LUCENE-8586) + * The ability to disable offset mangling in WordDelimiterGraphFilter + (https://issues.apache.org/jira/browse/LUCENE-8509) + * BM25Similarity no longer multiplies scores by k1 + 1 + +commit d2886e1c815cd697024c3ad533e58ef4aaa40751 +Author: Alan Woodward +Date: Wed Dec 5 12:41:42 2018 +0000 + + HLRC: execute watch API (#35868) + + This change adds support for the execute watch API in the high level rest client + +commit a264cb6ddb809b8d7ef9375a2caea5a6f2e628a4 +Author: Martijn van Groningen +Date: Wed Dec 5 13:39:14 2018 +0100 + + Refactor AutoFollowCoordinator to track leader indices per remote cluster (#36031) + + and replaced poll interval setting with a hardcoded poll interval. + The hard coded interval will be removed in a follow up change to make + use of cluster state API's wait_for_metatdata_version. + + Before the auto following was bootstrapped from thread pool scheduler, + but now auto followers for new remote clusters are bootstrapped when + a new cluster state is published. + + Originates from #35895 + Relates to #33007 + +commit 60e45cd81df1f3a35fa66cb9499e30059a4f8be0 +Author: Alpar Torok +Date: Wed Dec 5 14:20:01 2018 +0200 + + Testing conventions task part 2 (#36107) + + Closes #35435 + + - make it easier to add additional testing tasks with the proper configuration and add some where they were missing. + - mute or fix failing tests + - add a check as part of testing conventions to find classes not included in any testing task. + +commit 786697a4b248ec6a67bc187e828b07664fd64d61 +Author: Martijn van Groningen +Date: Wed Dec 5 13:14:51 2018 +0100 + + [HLRC] Added support for CCR Stats API (#36213) + + This change also adds documentation for the CCR Stats API. + + Relates to #33824 + +commit 11935cd480b4ff88d95d71ce77c779258437b4c6 +Author: Martijn van Groningen +Date: Wed Dec 5 13:14:10 2018 +0100 + + Replace Streamable w/ Writeable in BaseTasksResponse and subclasses (#36176) + + This commit replaces usages of Streamable with Writeable for the + BaseTasksResponse / TransportTasksAction classes and subclasses of + these classes. + + Note that where possible response fields were made final. + + Relates to #34389 + +commit 59b090017421be570d0215de39d21681537b6f4d +Author: Alpar Torok +Date: Wed Dec 5 14:06:11 2018 +0200 + + Upgrade to Gradle 5.0 (#34263) + +commit 42457b5960d6407d0e047f53ed947f0c2ecc268e +Merge: 70c361ea5a4 4c2e0d76f32 +Author: Yannick Welsch +Date: Wed Dec 5 11:39:38 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 4c2e0d76f32090c2731a1fa8a8a6b3ede828ccb5 +Author: Adrien Grand +Date: Wed Dec 5 11:28:12 2018 +0100 + + Fix test failures when using the `termvectors` endpoint with a type name. + +commit b9707c29a1dc74cfa0dc1b2b4a6fbcca4d725c6d +Author: Martijn van Groningen +Date: Wed Dec 5 08:41:27 2018 +0100 + + [CCR] Change get autofollow patterns API response format (#36203) + + The current response format is: + + ``` + { + "pattern1": { + ... + }, + "pattern2": { + ... + } + } + ``` + + The new format is: + + ``` + { + "patterns": [ + { + "name": "pattern1", + "pattern": { + ... + } + }, + { + "name": "pattern2", + "pattern": { + ... + } + } + ] + } + ``` + + This format is more structured and more friendly for parsing and generating specs. + This is a breaking change, but it is better to do this now while ccr + is still a beta feature than later. + + Follow up from #36049 + +commit 090d766f354a407962c2811fc66c186e3021f90a +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Dec 5 16:44:49 2018 +1100 + + [Kerberos] Find if port is available before using it for Kdc server (#36192) + + If the randomly selected port was already in use the Kerberos + tests would fail. This commit adds check to see if the network + port is available and if not continue to find one for KDC server. + If it does not find port after 100 retries it throws an exception. + + Closes #34261 + +commit 327253870124b37da7d86d93bc2136958004e790 +Author: Tim Vernum +Date: Wed Dec 5 15:51:50 2018 +1100 + + Make credentials mandatory when launching xpack/migrate (#36197) + + Made credentials mandatory for xpack migrate tool. + Closes #29847. + + The x-pack user and roles APIs aren't available unless security is enabled, so the tool should always be called with the -u and -p options specified. + +commit e825ad5dce5f7db9715a266dde62814557a94243 +Author: Christoph Büscher +Date: Wed Dec 5 00:31:52 2018 +0100 + + Add javadocs about expected exceptions to RestHighLevelClient (#36216) + + Add a more detailed section about what exceptions to expect from the blocking + calls in this class and removing the mostly redundant mentions of the + IOExceptions from each method javadoc since it doesn't give much details about + the expected exceptions anyway. + + Closes #30334 + +commit 8bde608979bd063168fcc511c84cbe0d96b9ff00 +Author: Tim Brooks +Date: Tue Dec 4 14:36:50 2018 -0700 + + Register CcrRepository based on settings update (#36086) + + This commit adds an empty CcrRepository snapshot/restore repository. + When a new cluster is registered in the remote cluster settings, a new + CcrRepository is registered for that cluster. + + This is implemented using a new concept of "internal repositories". + RepositoryPlugin now allows implementations to return factories for + "internal repositories". The "internal repositories" are different from + normal repositories in that they cannot be registered through the + external repository api. Additionally, "internal repositories" are local + to a node and are not stored in the cluster state. + + The repository will be unregistered if the remote cluster is removed. + +commit e1fb1505f94b8012d7a1d43f63fccda2dc9faad2 +Author: Lisa Cawley +Date: Tue Dec 4 13:18:54 2018 -0800 + + [DOCS] Moves security config file info (#36232) + +commit 70c361ea5a4b88a1e75752320f812b5aaf720a97 +Merge: 9026f98aca1 3c54b413adf +Author: Yannick Welsch +Date: Tue Dec 4 21:26:11 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 3c54b413adf53067ed6c174d872ce3650efc94db +Author: Armin Braun +Date: Tue Dec 4 20:53:51 2018 +0100 + + SNAPSHOT: Repo Creation out of ClusterStateTask (#36157) + + * Move `createRepository` call out of cluster state tasks + * Now only `RepositoriesService#applyClusterState` manipulates `this.repositories` + * Closes #9488 + +commit c007a42ba0a429f1fd550fb30f3bec2ec26e5948 +Author: Benjamin Trent +Date: Tue Dec 4 13:29:29 2018 -0600 + + [ML] adjusting post backport (#36222) + +commit d24b40f688935dc3c81f968b5d8a93b9905f1039 +Author: Adrien Grand +Date: Tue Dec 4 19:22:17 2018 +0100 + + Make typeless APIs usable with indices whose type name is different from `_doc` (#35790) + + This commit makes `document`, `update`, `explain`, `termvectors` and `mapping` + typeless APIs work on indices that have a type whose name is not `_doc`. + Unfortunately, this needs to be a bit of a hack since I didn't want calls with + random type names to see documents with the type name that the user had chosen + upon type creation. + + The `explain` and `termvectors` do not support being called without a type for + now so the test is just using `_doc` as a type for now, we will need to fix + tests later but this shouldn't require further changes server-side since passing + `_doc` as a type name is what typeless APIs do internally anyway. + + Relates #35190 + +commit 9026f98aca154699953f8873fe53d6b515ad73de +Author: Yannick Welsch +Date: Tue Dec 4 18:53:32 2018 +0100 + + Remove trace logging from CoordinatorTests + +commit 7b82c6c4cc5d344328e29d34eccf791b6b0203ab +Author: David Turner +Date: Tue Dec 4 16:59:08 2018 +0000 + + Override gateway.recover_after_master_nodes in test + + By default gateway.recover_after_master_nodes is set to + discovery.zen.minimum_master_nodes but in this Zen2 test this is set to an + unreasonably large value. This change updates it so the cluster can properly + form. + +commit 8ccb4660728739ce19d305a7529df5b21d70237d +Author: Martijn van Groningen +Date: Tue Dec 4 17:57:42 2018 +0100 + + [TEST] Remove auto follow pattern at the end of the test, + so that it does not collide with auto follow patterns in other tests. + +commit 0df08dd4587c8f9774dad348f7795c677d24fb4b +Author: Adrien Grand +Date: Tue Dec 4 17:53:20 2018 +0100 + + Set Lucene version upon index creation. (#36038) + + It is important that all shards of a given index have the same + `indexCreatedVersionMajor` to Lucene, or eg. merging those shards is going to + be considered illegal. At the moment, we use the latest Lucene version when + creating a shard, which could cause shards to have different created versions + eg. in case of forced allocation. This commit makes sure to reuse the + appropriate Lucene version in order to avoid such issues. + + Closes #33826 + +commit b59deb573e22743cf04fa5fa3df52d1583c9be3d +Author: Nhat Nguyen +Date: Tue Dec 4 11:15:34 2018 -0500 + + Always set soft-deletes field of IndexWriterConfig (#36196) + + Today we configure the soft-deletes field iff soft-deletes enabled. + Although this choice was correct, it prevents an engine with + soft-deletes disabled from opening a Lucene index with soft-deletes. + Moreover, this change should not have any side-effect if a Lucene index + does not have any soft-deletes. + + Relates #36141 + +commit 166d9a94d461ca8216b91dde2712a7912935041c +Author: Benjamin Trent +Date: Tue Dec 4 09:41:47 2018 -0600 + + [ML] Add lazy parsing for DatafeedConfig:Aggs,Query (#36117) + + * Lazily parsing aggs and query in DatafeedConfigs + + * Adding parser tests + + * Fixing exception types && unneccessary checked ex + + * Adding semi aggregation parser + + * Adding tests, fixing up semi-parser + + * Reverting semi-parsing + + * Moving agg validations + + * Making bad configs throw badRequestException + +commit 6e1ff31222ec8cc4c6270711ee5168d42ad633ee +Author: Martijn van Groningen +Date: Tue Dec 4 15:55:15 2018 +0100 + + [CCR] AutoFollowCoordinator should tolerate that auto follow patterns may be removed (#35945) + + AutoFollowCoordinator should take into account that after auto following + an index and while updating that a leader index has been followed, that + the auto follow pattern may have been removed via delete auto follow patterns + api. + + Also fixed a bug that when a remote cluster connection has been removed, + the auto follow coordinator does not die when it tries get a remote client for + that cluster. + + Closes #35480 + +commit 35e3d77e2cebd78d1b5f88dac5238e8b060d328f +Author: Andrey Ershov +Date: Tue Dec 4 14:45:45 2018 +0100 + + [Zen2] Implement state recovery (#36013) + + This commit implements proper metadata recovery for Zen2. + + GatewayService is responsible for the recovery. In Zen1 GatewayService + creates an instance of Gateway, that is used to reach out to other cluster + nodes, get their state and calculate the most up-to-date state based on + versions. After that Gateway performs upgrade and archival of + ClusterSettings and closes bad indices. Then recovered state is passed to GatewayService.GatewayRecoveryListener that mixes up current state + and restored state, removes state not recovered block, creates the + routing table and performs re-routing. + + In Zen2 we should perform this kind of logic on cluster startup, except + mixing state (because there is nothing to mix) and opening routing table. + + This commit refactors out all `ClusterUpdate` functions in a separate class + `ClusterStateUpdaters`, which is used by `Gateway` and `GatewayService` + in case of Zen1, and by `GatewayMetaState` and `GatewayService` in case of + Zen2. + + This commit also switches all integration tests that are already using Zen2 from + InMemoryPersistedState to GatewayMetaState. + +commit 5d684ca47351b54f1ebb2ac5b732d894127ec4bc +Author: Tanguy Leroux +Date: Tue Dec 4 14:39:31 2018 +0100 + + Combine the execution of an exclusive replica operation with primary term update (#36116) + + This commit changes how an operation which requires all index shard + operations permits is executed when a primary term update is required: + the operation and the update are combined so that the operation is + executed after the primary term update under the same blocking + operation. + + Closes #35850 + + Co-authored-by: Yannick Welsch + +commit 89fae428337f2ea18789166b0dd7233cae43b010 +Author: Guido Lena Cota +Date: Tue Dec 4 11:09:08 2018 +0100 + + (Minor) Fix some typos (#36180) + +commit 94010d33c968f2858115b58625f0ad4e2187fee5 +Author: Yannick Welsch +Date: Mon Dec 3 17:38:13 2018 +0100 + + Reenable BWC tests + +commit 80ee7943c9d091f1f747d109897e26c24dfe1499 +Merge: 034c7655b78 f8f521bad4b +Author: Yannick Welsch +Date: Tue Dec 4 09:37:09 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit d036e0ca89924b24860ca0b4fd9b442a4356f398 +Author: Alpar Torok +Date: Tue Dec 4 10:16:51 2018 +0200 + + Testclusters: implement starting, waiting for and stopping single cluster nodes (#35599) + +commit 034c7655b782d6006be7ac4dc0aa02970e8f17fe +Author: David Turner +Date: Tue Dec 4 07:48:56 2018 +0000 + + [Zen2] Reduce cluster scope in NodeDisconnectIT (#36168) + + This test suite can stop all the shared master-eligible nodes, which breaks the + cluster since any non-shared master-eligible nodes are stopped first in the + reset process between tests. + + Since this test suite can leave the cluster in this somewhat broken state, it + seems best that it uses a new cluster for each test. + +commit 579be9142e4782178ce5d5b40d79526d80e1b3e7 +Author: Martijn van Groningen +Date: Tue Dec 4 07:41:29 2018 +0100 + + [HLRC] Added support for CCR Get Auto Follow Pattern apis (#36049) + + This change also adds documentation for the Get Auto Follow Pattern API. + + Relates to #33824 + +commit 01b8f99c17c4fcbe5d6ab9c3bca9153f21e610ee +Author: Andy Bristol +Date: Mon Dec 3 16:07:16 2018 -0800 + + [test] mute RemoveCorruptedShardDataCommandIT + +commit a6ff9f7f66dcb4648a9d36cfd70bb6c7f495ea9c +Author: Andy Bristol +Date: Mon Dec 3 15:08:00 2018 -0800 + + [test] generate unique user names (#36179) + +commit 05d52b222f354f53d705a71beadc70f1bad9d1e3 +Author: lcawl +Date: Mon Dec 3 12:38:53 2018 -0800 + + [DOCs] More broken painless links + +commit e082cda0d65041dabf6c43a0c17ed5cab693e4b4 +Author: lcawl +Date: Mon Dec 3 12:17:26 2018 -0800 + + [DOCS] Fixes peer link + +commit d27aa72b17f1ceda156f705a262ef8c5f4106c35 +Author: João Barbosa +Date: Mon Dec 3 18:57:10 2018 +0000 + + Added soft limit to open scroll contexts #25244 (#36009) + + This change adds a soft limit to open scroll contexts that can be controlled with the dynamic cluster setting `search.max_open_scroll_context` (defaults to 500). + +commit fdec331b13dedf97d42b2e40d884f125a1668e4f +Author: Tal Levy +Date: Mon Dec 3 10:55:31 2018 -0800 + + [ILM] fix ilm.remove_policy rest-spec (#36165) + + The rest interface for remove-policy-from-index API does not support + `_ilm/remove`, it requires that an `{index}` pattern be defined in + the URL path. This fixes the rest-api-spec to reflect the implementation + +commit 19b936dcceae7a57fc3940ec4c3e936675f9a3b5 +Author: Alan Woodward +Date: Mon Dec 3 18:25:18 2018 +0000 + + Fix broken links in painless docs (#36170) + +commit 59ee8b5c691bec3e7cb784c7f2f390c0a7325896 +Author: Julie Tibshirani +Date: Mon Dec 3 10:22:42 2018 -0800 + + Remove the deprecated _termvector endpoint. (#36131) + +commit da2dbcdf38e02ae72a202aeaa78cdc5d006dae73 +Author: Alan Woodward +Date: Mon Dec 3 17:34:15 2018 +0000 + + Improve painless docs for score, similarity, weight and sort (#35629) + +commit c01aecb4b11447c6125095abdbc97c814b5dfc7c +Author: David Turner +Date: Mon Dec 3 17:19:18 2018 +0000 + + [Zen2] Do not probe non-master nodes back (#36160) + + Today if a node `A` sends a peers request to another node `B` then `B` will + react by sending a peers request back to `A`. However if `A` is not + master-eligible then this reaction is pointless and fails with an exception + saying `non-master-eligible node found`, adding noise to the logs. This change + suppresses this response to non-master-eligible nodes. + +commit f8f521bad4b28f060e30d963f41c8ff6f9e6e8cd +Author: Jake Landis +Date: Mon Dec 3 10:26:08 2018 -0600 + + Deprecate /_xpack/monitoring/* in favor of /_monitoring/* (#36130) + + This commit is part of our plan to deprecate and ultimately remove the + use of _xpack in the REST APIs. + + * Add deprecation for /_xpack/monitoring/_bulk in favor of /_monitoring/bulk + * Removed xpack from the rest-api-spec and tests + * Removed xpack from the Action name + * Removed MonitoringRestHandler as an unnecessary abstraction + * Minor corrections to comments + + Relates #35958 + +commit 433a506d06c73d227920261df06aa404de310e0b +Author: Armin Braun +Date: Mon Dec 3 16:39:29 2018 +0100 + + SNAPSHOT: Improve Resilience SnapshotShardService (#36113) + + * Resolve the index in the snapshotting thread + * Added test for routing table - snapshot state mismatch + +commit 9c1c46a02f8ba4b3850578e7f4764ad58d0515e4 +Author: Nhat Nguyen +Date: Mon Dec 3 08:55:48 2018 -0500 + + TEST: Adjust min_retained_seq_no expectation + + min_retained_seq_no is non-negative, however, if the number of retained + operations is greater than 0, then the expectation may be negative. + +commit b5cae0af58caaae726c6d980665ac95282e77bfb +Author: Luca Cavanna +Date: Mon Dec 3 13:55:18 2018 +0100 + + Enforce max_buckets limit only in the final reduction phase (#36152) + + Given that we check the max buckets limit on each shard when collecting the buckets, and that non final reduction cannot add buckets (see #35921), there is no point in counting and checking the number of buckets as part of non final reduction phases. + + Such check is still needed though in the final reduction phases to make sure that the number of returned buckets is not above the allowed threshold. + + Relates somehow to #32125 as we will make use of non final reduction phases in CCS alternate execution mode and that increases the chance that this check trips for nothing when reducing aggs in each remote cluster. + +commit 36ddca7d0ccf1af058213ffef1d29a0a2c696673 +Author: Boaz Leskes +Date: Mon Dec 3 13:45:19 2018 +0100 + + Disable merges in testReuseInFileBasedPeerRecovery + + The test assumes lucene files don't change. + + Closes #35772 + +commit 74aca756b80ff5ad5e633ab54f184cc3ec1ba7a0 +Author: Jim Ferenczi +Date: Mon Dec 3 11:49:11 2018 +0100 + + Remove the distinction between query and filter context in QueryBuilders (#35354) + + When building a query Lucene distinguishes two cases, queries that require to produce a score and queries that only need to match. We cloned this mechanism in the QueryBuilders in order to be able to produce different queries based on whether they need to produce a score or not. However the only case in es that require this distinction is the BoolQueryBuilder that sets a different minimum_should_match when a `bool` query is built in a filter context.. + This behavior doesn't seem right because it makes the matching of `should` clauses different when the score is not required. + + Closes #35293 + +commit 328d022ddd8b14e31a8e89dd0becd22136ed04cb +Author: Armin Braun +Date: Mon Dec 3 11:21:42 2018 +0100 + + MINOR: Some Cleanups around Store (#36139) + + * Moved method `canOpenIndex` is only used in tests -> moved to test CP + * Simplify `org.elasticsearch.index.store.Store#renameTempFilesSafe` + * Delete some dead methods + +commit f763037b03d9a9d76575bb94efbe40b31496203e +Author: Armin Braun +Date: Mon Dec 3 11:11:10 2018 +0100 + + MINOR: BlobstoreRepository Cleanups (#36140) + + * Removed redundant private getter + * Removed unused `version` field + +commit 9c49aacbcf00ddeed73a02f6d9ba3bfc23595720 +Author: Armin Braun +Date: Mon Dec 3 10:02:35 2018 +0100 + + MINOR: Remove Dead Code in QueryCache (#36147) + +commit fa4d5f844d4b05de74ab4e3b79956cedb9a921a6 +Author: Alpar Torok +Date: Mon Dec 3 11:01:05 2018 +0200 + + Fix test fixtures on aufs (#36105) + + Closes #36073 + + The problem showed up on debian 8 which uses aufs docker storage + driver by default as opposed to overlay2 used on other distros. + aufs does not support acls and thus the failure. + The --use-ntvfs option instructs samba not to rely on acls. + From what I can tell this is an implementation detail that should not + affect the tests ( which continue to pass ) + +commit 6a773d7d511546661e0bc6cf3e84cfa1152cdb85 +Author: Dimitrios Liappis +Date: Mon Dec 3 10:43:36 2018 +0200 + + Fix error message when package install fails due to missing Java (#36077) + + Currently is `java` is not in $PATH the preinst script fails + prematurely and prevents an appropriate message from getting displayed + to the user. + + Make package installation more user friendly when java is not in + $PATH and add a test for it. + + Also use a she-bang in the preinst script, as, at least in Debian, + maintainer scripts must start with the #! convention [1]. + + Relates #31845 + + [1] https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html + +commit 43773a32a47ab58a9683ada6bf738430fc5c1cd4 +Author: Martijn van Groningen +Date: Mon Dec 3 08:04:29 2018 +0100 + + Replace Streamable w/ Writeable in BaseTasksRequest and subclasses (#35854) + + * Replace Streamable w/ Writeable in BaseTasksRequest and subclasses + + This commit replaces usages of Streamable with Writeable for the + BaseTasksRequest / TransportTasksAction classes and subclasses of + these classes. + + Relates to #34389 + +commit 9c0a429709c4f32dae0d2a7f19d9b723904ff63b +Author: Armin Braun +Date: Mon Dec 3 06:16:12 2018 +0100 + + TESTS: Fix IndexStatsIT#testFilterCacheStats (#36143) + + * Test randomly failed because of background merges + * Fixed by force merging down to a single segment + * Closes #32506 + +commit d20bb3789d050568a0453fdcb4402c52e1d4f31f +Author: Tim Vernum +Date: Mon Dec 3 10:05:57 2018 +1100 + + Add DEBUG/TRACE logs for LDAP bind (#36028) + + Introduces a debug log message when a bind fails and a trace message + when a bind succeeds. + + It may seem strange to only debug a bind failure, but failures of this + nature are relatively common in some realm configurations (e.g. LDAP + realm with multiple user templates, or additional realms configured + after an LDAP realm). + +commit 8011438ea8878032af7abc4f06847c62fca0b845 +Author: David Turner +Date: Sun Dec 2 09:39:43 2018 +0000 + + Use correct source of randomness + + This fixes a failure of InternalTestClusterTests#testBeforeTest which checks + that the cluster is set up the same when starting from the same seed. Trappily, + using ESTestCase#randomIntBetween() is no good, we have to use + InternalTestCluster#random via RandomNumbers#randomIntBetween() instead. + +commit 8bb19529754652a173e8cf6519c3eaf9e9b31a93 +Author: David Turner +Date: Sat Dec 1 09:54:01 2018 +0000 + + Fix NodeJoinTests again (#36133) + + In #36033 we removed a catch block because we thought we were preventing + exceptions by avoiding concurrent elections, missing the obvious fact that some + joins are supposed to be failing. + + As a quick fix the catch was reinstated in 3a5dab6d8ef23ba3cb60162e9e2b7e002145 + but this change adds finesse by only catching exceptions from the joins that we + expect to fail. It also inlines an always-false parameter to `initialState()`. + +commit 9cc416bc4611d52ce11297b94a46053fa38df5eb +Author: David Turner +Date: Sat Dec 1 07:20:19 2018 +0000 + + Weaken assertion in PeerFinder + + It can be inactive with no leader if it's handling an incoming PeersRequest + before being activated for the first time. + +commit 3a5dab6d8ef23ba3cb60162e9e2b7e002145dc58 +Author: David Turner +Date: Sat Dec 1 07:10:19 2018 +0000 + + Reinstate catch removed in error in #36033 + +commit 8191348d6bca2fcdb3c3999f80894dc92bb31fdb +Author: David Turner +Date: Sat Dec 1 06:43:11 2018 +0000 + + [Zen2] Only bootstrap a single node (#36119) + + Today, we allow all nodes in an integration test to bootstrap. However this + seems to lead to test failures due to post-election instability. The change + avoids this instability by only bootstrapping a single node in the cluster. + +commit 46962308aa9eb274231ba79ce295baad54ce015a +Author: Lisa Cawley +Date: Fri Nov 30 16:58:19 2018 -0800 + + [DOCS] Replace deprecated ldap setting (#36022) + +commit 0e1ddfd825df21adf641bd96c7e6d5c6ca8e0738 +Author: Julie Tibshirani +Date: Fri Nov 30 15:11:29 2018 -0800 + + Deprecate types in document delete requests. (#36087) + + * Make sure to use _doc as a type name in the CRUD HLRC tests. + * Deprecate types in document delete requests. + +commit 7b999bdc88c31eebb7d00b7adcadcfdac9f892a0 +Author: Jay Modi +Date: Fri Nov 30 14:38:10 2018 -0700 + + Fix logic in dockerComposeSupported (#36125) + + The logic in the dockerComposeSupported method currently returns false + even when docker and docker compose are available on the build machine. + This change updates the check to see if docker compose is available in + one of the two paths and allows the `tests.fixture.enabled` property to + disable the tests even if docker compose is available. + +commit 98b290637dabf19addc708221ee34ca0c9476154 +Author: Julie Tibshirani +Date: Fri Nov 30 13:11:58 2018 -0800 + + Deprecate the _termvector endpoint. (#36098) + +commit df56f0734ee1502bd55d315d5c6c81da7dd558d7 +Author: Nik Everett +Date: Fri Nov 30 16:06:58 2018 -0500 + + Tasks: Retry if task can't be written (#35054) + + Adds about a minute worth of backoffs and retries to saving task + results so it is *much* more likely that a busy cluster won't lose task + results. This isn't an ideal solution to losing task results, but it is + an incremental improvement. If all of the retries fail when still log + the task result, but that is far from ideal. + + Closes #33764 + +commit 3c4953f4d1efac5f67a2e5cb3adf3e2a2de12912 +Author: Gordon Brown +Date: Fri Nov 30 13:05:14 2018 -0700 + + State default shard limit is not a recommendation (#36093) + + The new limit on the number of open shards in a cluster may be + interpreted by users as a sizing recommendation, but it is not. This + clarifies in the documentation that this is a safety limit, not a + recommendation. + +commit 0ebc17743adc1c04a561e4f30f188bbbf4c6629b +Author: Luca Cavanna +Date: Fri Nov 30 20:33:09 2018 +0100 + + Histogram aggs: add empty buckets only in the final reduce step (#35921) + + Empty buckets don't need to be added when performing an incremental reduction step, they can be added later in the final reduction step. This will allow us to later remove the max buckets limit when performing non final reduction. + +commit d7652963b1974778ff94a9c3a8876eb7e18d8b47 +Author: Gordon Brown +Date: Fri Nov 30 12:06:48 2018 -0700 + + Add note about ILM and Snapshots (#36023) + + This commit documents how Index Lifecycle Management + interacts with snapshot/restore, and documents a workaround + for situations in which ILM should not immediately resume + managing an index after it is restored. + +commit ea7ea51050d38b55be5827b241da2f9acea3486c +Author: Tim Brooks +Date: Fri Nov 30 11:30:42 2018 -0700 + + Make `TcpTransport#openConnection` fully async (#36095) + + This is a follow-up to #35144. That commit made the underlying + connection opening process in TcpTransport asynchronous. However the + method still blocked on the process being complete before returning. + This commit moves the blocking to the ConnectionManager level. This is + another step towards the top-level TransportService api being async. + +commit 465a65aa57336e06afd75139cf119d2db419ef56 +Author: Chris Koehnke +Date: Fri Nov 30 13:17:39 2018 -0500 + + Docs: Fix release-state check for oss repositories (#36120) + + To get the newly added oss apt/yum sections to get rendered for + `released` and `prerelease` versions the condition needs to be modified. + +commit 986bf52d1f3a3dc38ee4a29e1c8b3f5c7ef55126 +Author: Armin Braun +Date: Fri Nov 30 18:53:08 2018 +0100 + + [Zen2] Allow Setting a List of Bootstrap Nodes to Wait for (#35847) + +commit e179fd1274331e30bc3ddfb1ab789fb0d38c92bd +Author: Jim Ferenczi +Date: Fri Nov 30 18:02:37 2018 +0100 + + Add support for rest_total_hits_as_int in watcher (#36035) + + This change adds the support for rest_total_hits_as_int + in the watcher search inputs. Setting this parameter in the request + will transform the search response to contain the total hits as + a number (instead of an object). + Note that this parameter is currently a noop since #35849 is not + merged. + + Closes #36008 + +commit c24be278e4d5b706f51587f8da002915c038a246 +Author: Lisa Cawley +Date: Fri Nov 30 08:55:29 2018 -0800 + + [DOCS] Refreshes population job examples (#36101) + +commit 54facbe3258b1b9196b9b7a2384fb7d2d2338d39 +Author: Jim Ferenczi +Date: Fri Nov 30 17:19:46 2018 +0100 + + [TEST] fix typo in get-watch documentation (bis) + +commit 26dcbcc8cce2b871d04309aaae43f2bafec7e072 +Author: Tim Brooks +Date: Fri Nov 30 09:04:51 2018 -0700 + + Remove `MockTcpTransport` for ESIntegTestCase (#36089) + + This commit removes the `MockTcpTransport` as a transport option for + `ESIntegTestCase`. It is the first step in replacing the usages of + `MockTcpTransport` with `MockNioTransport`. + +commit da100c5479ba3796a52618f0dd12009bc674fc35 +Author: Tim Brooks +Date: Fri Nov 30 09:04:32 2018 -0700 + + Remove `Lifecycle` from `ConnectionManager` (#36092) + + Prior to #35441 `ConnectionManager` had a `Lifecycle` object to support + the ping runnable. After that commit, the connection amanger only needs + the existing `AtomicBoolean` to indicate if it is running. + +commit 370472b6d1aa0651db78d5873d07ca169d0163dc +Author: Tim Brooks +Date: Fri Nov 30 09:02:10 2018 -0700 + + Upgrade Netty 4.3.32.Final (#36102) + + This commit upgrades netty. This will close #35360. Netty started + throwing an IllegalArgumentException if a CompositeByteBuf is + created with < 2 components. Netty4Utils was updated to reflect this + change. + +commit 11fa5c626b32cf7c30dac0550023e61cc1ba1875 +Author: Jim Ferenczi +Date: Fri Nov 30 16:05:59 2018 +0100 + + [TEST] Fix random test failure in GetWatchResponseTests + +commit acdf9666d5e793495f8351001b9edc573c561436 +Author: Christophe Bismuth +Date: Fri Nov 30 16:10:13 2018 +0100 + + Add `minimum_should_match` section to the query_string docs + + Closes #34142 + +commit 08b9e313737f40ebe31644a37986d1a4c3a117d0 +Author: Jim Ferenczi +Date: Fri Nov 30 14:40:09 2018 +0100 + + [TEST] fix link in get-watch documentation + +commit bb2cf7e6be2a92a954f596b5e241c81415d1310e +Author: patrykk21 <39259934+patrykk21@users.noreply.github.com> +Date: Fri Nov 30 14:30:23 2018 +0100 + + [Docs] Clarify search_after behavior + + Closes #34232 + +commit 43ea498f2fadb94ce58611c7a4a3f0dd147af2fe +Author: Luca Cavanna +Date: Fri Nov 30 14:15:52 2018 +0100 + + [TEST] Reduce number of buckets created in InternalDateHistogramTests + + New that we test with min_doc_count set to 0 as well, we may end up generating a lot more buckets. This commit adjusts the min bound and max bound, as well as the offset for each randomly generated agg instance so that we don't end up hitting the 10.000 max buckets limit. + + Relates to #36064 + +commit 6d4dfef64ea74fb6ba42c4ae99ec114d1a96372b +Author: Alpar Torok +Date: Fri Nov 30 15:16:23 2018 +0200 + + Conditional conffiles for packages (#36046) + + Relates to #35810 + +commit 5e6460acb3025c806f52a0a93d8e3045fd15ed0f +Author: Jim Ferenczi +Date: Fri Nov 30 12:53:04 2018 +0100 + + [TEST] fix typo in get-watch documentation + +commit e3eb05b14b77d5dd0d9b831a315ffa4f35ca1eb4 +Author: Luca Cavanna +Date: Fri Nov 30 11:21:46 2018 +0100 + + [TEST] Increase InternalDateHistogramTests coverage (#36064) + + In this test we were randomizing different values but minDocCount was hardcoded to 1. It's important to test other values, especially `0` as it's the default. The test needed some adapting in the way buckets are randomly generated: all aggs need to share the same interval, minDocCount and emptyBucketInfo. Also assertions need to take into account that more (or less) buckets are expected depending on minDocCount. + +commit 32f7fbd9f0429dbf80d57778c6e55a50d0938f85 +Author: Martijn van Groningen +Date: Fri Nov 30 10:58:46 2018 +0100 + + [TEST] Set 'index.unassigned.node_left.delayed_timeout' to 0 in ccr tests + + Some tests kill nodes and otherwise it would take 60s by default + for replicas to get allocated and that is longer than we wait + for getting in a green state in tests. + + Relates to #35403 + +commit 5c7b2c5f9bec850ea68f3daaf229e7f18df94c47 +Author: Jim Ferenczi +Date: Fri Nov 30 11:02:46 2018 +0100 + + HLRC: Add get watch API (#35531) + + This changes adds the support for the get watch API in the high level rest client. + +commit fa3d365ee83a089d461eacf66f2118f68dc1a8ed +Author: Adrien Grand +Date: Fri Nov 30 10:32:46 2018 +0100 + + Fix CompositeBytesReference#slice to not throw AIOOBE with legal offsets. (#35955) + + CompositeBytesReference#slice has two bugs: + - One that makes it fail if the reference is empty and an empty slice is + created, this is #35950 and is fixed by special-casing empty-slices. + - One performance bug that makes it always create a composite slice when + creating a slice that ends on a boundary, this is fixed by computing `limit` + as the index of the sub reference that holds the last element rather than + the next element after the slice. + + Closes #35950 + +commit 54cf1f9d7442686f9cdc468f43e28373001ca7c3 +Author: Dimitris Athanasiou +Date: Fri Nov 30 09:25:35 2018 +0000 + + [ML] Refactor control message writer to allow reuse for other processes (#36070) + +commit d128b381167144fa9cdbf3d9cdfdb882dae2501b +Author: Alpar Torok +Date: Fri Nov 30 09:34:53 2018 +0200 + + Introducing testing conventions task (#35861) + +commit 364badd12e17d05d4e93247427877bec4270ce7d +Author: Martijn van Groningen +Date: Fri Nov 30 07:41:44 2018 +0100 + + Fix serialization bug in painless execute api request (#36075) + + The `xContentType` was incorrectly serialized: + `xContentType.mediaTypeWithoutParameters()` was used to serialize, but + `xContentType.mediaType()` was used to de-serialize. + + Also the serialization test class did not test `ContextSetup` well, + this was due to limitation in serialization test base class. Changed the + test class to manually test xcontent serialization, so that for both binary + and xcontent serialization tests, the `ContextSetup` is properly tested. + + Closes #36050 + +commit 63a9571c4c54f1f1573de3351d1485ab86173f28 +Author: Peter Dyson +Date: Fri Nov 30 14:56:32 2018 +1000 + + Explain nondeterministic template merging (#35405) + +commit 14eeb2aa30d4c4456acc44a0b5923867ecf95d70 +Author: Ryan Ernst +Date: Thu Nov 29 18:58:00 2018 -0800 + + Build: Remove padding from java version in build info (#36088) + + This commit removes padding from the java version in the output of + compiler/runtime/gradle java versions. This value is only ever 1 or 2 + digits, and the later information (hotspot/jvm vendor info) is separated + by the java home path from the other versions, so there isn't a visual + reason for needing exact vertical alignment. + +commit 78e07d467c87e4728734b0da1ebc50f43109175e +Author: Tal Levy +Date: Thu Nov 29 14:11:51 2018 -0800 + + [ILM] rest-spec for remove-policy had wrong link (#36083) + +commit d567f89719dd6f938f0cf2ed037477672b5946a0 +Author: Gordon Brown +Date: Thu Nov 29 14:09:51 2018 -0700 + + Recommend index templates when using ILM Rollover (#35922) + + Because rollover does not propogate the ILM policy in use, add a note to + the docs recommending the use of index templates when using rollover. + +commit 9ef54203d73498a60674b75e5cf1d0c9c216116c +Author: Ryan Ernst +Date: Thu Nov 29 11:54:59 2018 -0800 + + Packaging: Add vagrant tests for windows service (#33729) + + This commit adds tests using the vagrant windows image for the windows + service manager script. + +commit 04ee05f199855b3f0a097db62148f884fe4161bc +Author: Albert Zaharovits +Date: Thu Nov 29 21:34:16 2018 +0200 + + Fix Watcher NotificationService's secure settings (#35610) + + The NotificationService (base class for SlackService, HipchatService ...) has both dynamic + cluster settings and SecureSettings and builds the clients (Account) that are used to comm + with external services. This commit fixes an important bug about updating/reloading any + of these settings (both Secure and dynamic cluster). Briefly the bug is due to the fact that + both the secure settings as well as the dynamic node scoped ones can be updated + independently, but when constructing the clients some of the settings might not be visible. + +commit fa5f551b8034da55b30a736d555c9d3777b97e6d +Author: Tal Levy +Date: Thu Nov 29 11:11:20 2018 -0800 + + [ILM] make alias swapping atomic (#35972) + + In the current implementation, there is a time between the + ShrinkStep and the ShrinkSetAliasStep that both the source and + target indices will be present with the same aliases. This means + that queries to during this time will query both and return + duplicates. This fixes that scenario by moving the alias inheritance + to the same aliases update request as is done in ShrinkSetAliasStep + +commit ba3ee98943a79f04c3b04682db11a3dac3a9e36f +Author: Jay Modi +Date: Thu Nov 29 11:37:06 2018 -0700 + + Security: improve exact index matching performance (#36017) + + This commit improves the efficiency of exact index name matching by + separating exact matches from those that include wildcards or regular + expressions. Internally, exact matching is done using a HashSet instead + of adding the exact matches to the automata. For the wildcard and + regular expression matches, the underlying implementation has not + changed. + +commit 61c2db5ebbec9b32a02be79a8adce4728295b207 +Author: Zachary Tong +Date: Thu Nov 29 12:58:23 2018 -0500 + + Revert "Deprecate X-Pack centric rollup endpoints (#35962)" + + This reverts commit b84f1f6a3ad511538ea102badfefa934fa8f0336. + +commit 40c54454805a468c2bc5d02073044addb68e8c21 +Author: Zachary Tong +Date: Thu Nov 29 12:56:25 2018 -0500 + + Revert "[TEST] Use deprecated form of rollup endpoint in mixed cluster (#36000)" + + This reverts commit 85cdf4f9133190d9e0fbb7dc8b791262571cab52. + +commit f8636e58f97ee965940082a4ad899854e8a70cb9 +Author: Jake Landis +Date: Thu Nov 29 11:45:45 2018 -0600 + + Support content type `application/x-ndjson` in DeprecationRestHandler (#36025) + + org.elasticsearch.rest.RestController#hasContentType checks to see if the + RestHandler supports the `application/x-ndjson` Content-Type. DeprecationRestHandler + is a wrapper around the real RestHandler, and prior to this change + would always return `false` due to the interface's default supportsContentStream(). + This prevents API's that use multi-line JSON from properly being deprecated + resulting in an HTTP 406 error. + + This change ensures that the DeprecationRestHandler honors the + supportsContentStream() of the wrapped RestHandler. + + Relates to #35958 + +commit 95d9cefea745b834e6ab7e8cefff177bfb1b0e47 +Author: Ryan Ernst +Date: Thu Nov 29 09:39:17 2018 -0800 + + Build: Fix jdbc jar pom to not include deps (#36036) + + This commit adds back the exclusion of dependencies from the pom file + for the jdbc jar, which was accidentally lost in #32014 + +commit 8a7f3f75f3eefde29fed0d0d414a798c50a25ee6 +Author: Jim Ferenczi +Date: Thu Nov 29 18:36:16 2018 +0100 + + Add support for rest_total_hits_as_int (#36051) + + The support for rest_total_hits_as_int has already been merged to 6x + in #35848 so this change adds this new option to master. The plan was + to add this new option as part of #35848 but we've decided to wait a few + days before merging this breaking change so this commit just handles + the new option as a noop exactly like 6x for now. This will allow + users to migrate to this parameter before #35848 is merged. + + Relates #33028 + +commit 8006068873b96739f45f98ecdc027e2ea488f33d +Author: Toby McLaughlin +Date: Fri Nov 30 04:33:08 2018 +1100 + + [DOCS] Remove "platinum" references for Docker TLS (#35890) + +commit 1d458e3f60899e59f0a6b9272401287c69ed6d0f +Author: Costin Leau +Date: Thu Nov 29 18:52:47 2018 +0200 + + SQL: Make INTERVAL millis optional (#36043) + + Fractions of the second are not mandatory anymore inside INTERVAL + declarations + + Fix #36032 + +commit c305f9dc03eb089e3f7dc16cd4e3ddd546222360 +Author: Tim Brooks +Date: Thu Nov 29 08:55:53 2018 -0700 + + Make keepalive pings bidirectional and optimizable (#35441) + + This is related to #34405 and a follow-up to #34753. It makes a number + of changes to our current keepalive pings. + + The ping interval configuration is moved to the ConnectionProfile. + + The server channel now responds to pings. This makes the keepalive + pings bidirectional. + + On the client-side, the pings can now be optimized away. What this + means is that if the channel has received a message or sent a message + since the last pinging round, the ping is not sent for this round. + +commit 5eb704040fd09e10132e430f49e78df00d3aaf56 +Author: Albert Zaharovits +Date: Thu Nov 29 16:12:25 2018 +0200 + + While most peoples' opinions change, the conviction of their correctness never does. (#35988) + +commit 9709282735820fd2584c7fa8d1cae9a50975555b +Author: Alpar Torok +Date: Thu Nov 29 15:54:34 2018 +0200 + + Don't disable tasks based on the plugin (#36045) + + Some times the test fixtures plugin did not correctly disable tasks + from the build plugin as it should. + The plugin manager and tasks both use domain name collections so + the previus conde should have worked. + I did not have trime to track it down, but suspect there's some race + condition in Gradle causing this. The plugin manager is still incubating. + Since the tasks are on the cp even if the plugin is not applyed, we + don't really need to involve the plugin at all. + + Closes #36041 + +commit 4b0b479d1bd2a5bcd866837f079689dd782ff567 +Author: Nik Everett +Date: Thu Nov 29 08:38:09 2018 -0500 + + Build: Fix reproduce info for methods with ( or ) (#35712) + + Our `REPRODUCE WITH` line wasn't working for tests with `(` or `)` in + the method name. This isn't super common, but happens sometimes for our + yml based rest tests. This is because randomized runner's globbing + mechanism dind't escape `(` or `)`. I filed this upstream at + https://github.com/randomizedtesting/randomizedtesting/issues/271 + and Dawid kindly fixed the issue. This upgrades to pick up the fix. + + Closes #35692 + +commit ecd29089a81bfe534676b9eeb55f54908d107118 +Author: Jim Ferenczi +Date: Thu Nov 29 14:35:25 2018 +0100 + + Cache the score of the parent document in the nested agg (#36019) + + The nested agg can defer the collection of children if it is nested + under another aggregation. In such case accessing the score in the children + aggregation throws an error because the scorer has already advanced to the next + parent. This change fixes this error by caching the score of the parent in the + nested aggregation. Children aggregations that work on nested documents will be + able to access the _score. Also note that the _score in this case is always the + parent's score, there is no way to retrieve the score of a nested docs in aggregations. + + Closes #35985 + Closes #34555 + +commit 48dc6c3442e85d25366013961cfe249fabd18212 +Author: Armin Braun +Date: Thu Nov 29 14:34:10 2018 +0100 + + [Zen2] Implement Tombstone REST APIs (#36007) + + * [Zen2] Implement Tombstone REST APIs + + * Adds REST API for withdrawing votes and clearing vote withdrawls + * Tests added to Netty4 module since we need a real Network impl. for Http endpoints + +commit 85cdf4f9133190d9e0fbb7dc8b791262571cab52 +Author: Zachary Tong +Date: Thu Nov 29 07:37:33 2018 -0500 + + [TEST] Use deprecated form of rollup endpoint in mixed cluster (#36000) + + When wiping rollup jobs, if we are in a mixed cluster with < v7.0 nodes + we need to fall back to the deprecated endpoint because we may talk + to a 6.x node. + +commit 7f257187aff8382d6d48469d0674b3b4fcfac4ea +Author: David Turner +Date: Thu Nov 29 12:18:35 2018 +0000 + + [Zen2] Update default for USE_ZEN2 to true (#35998) + + Today the default for USE_ZEN2 is false and it is overridden in many places. By + defaulting it to true we can be sure that the only places in which Zen2 does + not work are those in which it is explicitly set to false. + +commit 277ccba3bd2e80df981e7c6001cd8e47d9e6d631 +Author: David Turner +Date: Thu Nov 29 12:11:08 2018 +0000 + + [Zen2] fix NodeJoinTests#testConcurrentJoining() (#36033) + + Today we sometimes create a setup in which the node is a quorum on its own, + which allows it to win a pre-voting round and schedule an election essentially + at will, causing it to discard all the joins it just received and fail the + test. This change excludes this case, preventing stray elections from ruining + things. + +commit 87408b04d4d1e780f5572ebb46788b116176738a +Author: David Turner +Date: Thu Nov 29 12:10:43 2018 +0000 + + [Zen2] Only elect master-eligible nodes (#35996) + + Today any node can win an election. However, the whole point of + master-eligibility is that master-ineligible nodes should not be elected as the + leader; furthermore master-ineligible nodes do not have any outgoing STATE + channels so cannot publish cluster states, so their leadership is ineffective + and disruptive. + + This change ensures that the elected leader is master-eligible by preventing + master-ineligible nodes from scheduling an election. + +commit 9fa9e1419f87946115a2b8812f2e4410e23d3afa +Author: Jason Tedor +Date: Thu Nov 29 07:09:37 2018 -0500 + + Remove X-Pack centric graph endpoints (#36010) + + This commit is part of our plan to deprecate and ultimately remove the + use of _xpack in the REST APIs. + +commit 525c3b044aa948742c0752365bb62f26d0c9eb20 +Author: Andrei Stefan +Date: Thu Nov 29 14:06:48 2018 +0200 + + Leftover from https://github.com/elastic/elasticsearch/pull/35964 (#36030) + +commit 89c515175b3c6f11989ac981df320dd700965f05 +Author: Ioannis Kakavas +Date: Thu Nov 29 13:28:45 2018 +0200 + + Update version for authenticate API responses + + This commits changes the serialization version from V_7_0_0 to + v_6_6_0 for the authenticate API response now that the work to add + the realm info in the response has been backported to 6.x in + b515ec7c9b9074dfa2f5fd28bac68fd8a482209e + + Relates #35648 + +commit c412ecd4a21de3898508ab1f5e44cdbdf55c258d +Author: Christoph Büscher +Date: Thu Nov 29 11:43:18 2018 +0100 + + Muting ClusterClientITT#testClusterHealthYellowClusterLevel + +commit a646f85a999c087668ef088387f1b918f4f3a8a3 +Author: Alan Woodward +Date: Thu Nov 29 10:35:38 2018 +0000 + + Ensure TokenFilters only produce single tokens when parsing synonyms (#34331) + + A number of tokenfilters can produce multiple tokens at the same position. This + is a problem when using token chains to parse synonym files, as the SynonymMap + requires that there are no stacked tokens in its input. + + This commit ensures that when used to parse synonyms, these tokenfilters either produce + a single version of their input token, or that they throw an error when mappings are + generated. In indexes created in elasticsearch 6.x deprecation warnings are emitted in place + of the error. + + * asciifolding and cjk_bigram produce only the folded or bigrammed token + * decompounders, synonyms and keyword_repeat are skipped + * n-grams, word-delimiter-filter, multiplexer, fingerprint and phonetic throw errors + + Fixes #34298 + +commit c63d0af913f16f83b0e06f90985182e40789f7bc +Author: Alpar Torok +Date: Thu Nov 29 11:14:44 2018 +0200 + + Prevent random build failures + + Looks like some odd race condition causes failed builds by attempting to + run the task that should be disabled. + Disable the task explicitly untill we figure it out. + +commit 1390f366d479d3c496ec80a4c065848531bd19f4 +Author: Martijn van Groningen +Date: Thu Nov 29 09:46:09 2018 +0100 + + [CCR] Only auto follow indices when all primary shards have started (#35814) + + This change adds an extra check that verifies that all primary shards + have been started of an index that is about to be auto followed. + + If not all primary shards have been started for an index + then the next auto follow run will try to follow to auto follow + this index again. + + Closes #35480 + +commit fbdfec43056d5e80aaf72b5db2300e5efc8b5ce7 +Author: Ryan Ernst +Date: Thu Nov 29 00:22:10 2018 -0800 + + Docs: Add note about oss repositories for deb/rpm (#35973) + + This commit adds a note about configring the yum/apt repositories for + oss only packages. + + closes #35960 + +commit 096762064170d9efdb1e8df928687dad7e4897a4 +Author: Tanguy Leroux +Date: Thu Nov 29 09:08:30 2018 +0100 + + ActiveShardCount should not fail when closing the index (#35936) + + The ActiveShardCount is used by cluster state observers to wait for a + given number of shards to be active before returning to the caller. The + current implementation does not work when an index is closed while an + observer is waiting on shards to be active. In this case, a NPE is thrown + and the observer is never notified that the shards won't become active. + + This commit fixes the ActiveShardCount.enoughShardsActive() so that it + does not fail when an index is closed, similarly to what is done when an + index is deleted. + +commit 45db829039d7b12b0a0dc44b1e0e5a0fcaeda719 +Author: Alpar Torok +Date: Thu Nov 29 09:43:16 2018 +0200 + + Replace fixtures with docker-compose (#35651) + + Creates a new plugin to manage docker-compose based test fixtures. + Convert the smb-fixture as a first example. + +commit 609f742e5f5d788cc496381c867403f49d735363 +Author: Tim Vernum +Date: Thu Nov 29 18:06:52 2018 +1100 + + Fix kerberos setting registration (#35986) + + In #30241 Realm settings were changed, but the Kerberos realm settings + were not registered correctly. This change fixes the registration of + those Kerberos settings. + + Also adds a new integration test that ensures every internal realm can + be configured in a test cluster. + + Also fixes the QA test for kerberos. + + Resolves: #35942 + +commit 6d01170ca1a471f296d0cfd303d0a0143f18de0d +Author: Martijn van Groningen +Date: Thu Nov 29 08:06:31 2018 +0100 + + [HLRC] Added support for CCR Delete Auto Follow Pattern API (#35981) + + This change also adds documentation for the Delete Auto Follow Pattern API. + + Relates to #33824 + +commit 93ed8b7d61bf78f8958877c3bce1554cd36d7712 +Author: Ignacio Vera +Date: Thu Nov 29 07:52:56 2018 +0100 + + HLRC: Add delete user action (#35294) + + * HLRC: Add delete user action + + It adds delete user action to the high level rest client. + + Relates #29827 + +commit e0a678f0c439ab30aaa8508a932d4888557a04a6 +Author: Alpar Torok +Date: Thu Nov 29 08:41:39 2018 +0200 + + Remove version.qualified from MainResponse (#35412) + + The fully qualified version will be returned as `version.number` + +commit afd42df15f55d16b650c60bf247e763777d5b181 +Author: Ryan Ernst +Date: Wed Nov 28 21:20:44 2018 -0800 + + Core: Deguice RepositoriesService (#36016) + + This commit moves the RepositoriesService to be created outside of + guice. + +commit 465233787ea34aa139d19bd11b612a1cc2b564cf +Author: Jason Tedor +Date: Wed Nov 28 16:55:59 2018 -0500 + + Move creation of temporary directory to Java (#36002) + + In the long run we want to move all of startup to a Java program. This + will simplify our startup scripts and make maintenance of startup less + dependent on the underlying platform that we run on. This commit moves + the creation of the temporary directory off of system-dependent commands + and onto a simple Java program. + +commit 0916a1fb640ab8ce0b8a81d89fccb315bfe5cc8b +Author: ik +Date: Thu Nov 29 06:29:05 2018 +0900 + + LLREST: Add PreferHasAttributeNodeSelector (#36005) + + `PreferHasAttributeNodeSelector` works like exactly like + `HasAttributeNodeSelector` but if not nodes match the attribute + then it will not filter the list of nodes. + +commit 906421cdad35536f58753d2f68380af7f4312d37 +Author: Lisa Cawley +Date: Wed Nov 28 12:36:29 2018 -0800 + + [DOCS] Adds higher-level monitoring configuration page (#35926) + +commit c26af3b0a2d513762297f69d9c5956d8ff398f97 +Author: Gordon Brown +Date: Wed Nov 28 13:19:33 2018 -0700 + + Deprecate X-Pack centric Migration endpoints (#35976) + + This commit is part of our plan to deprecate and remove the use of + _xpack in the REST API routes. + +commit 9ca3a06475efca454bdc82652b89545ec8c093d2 +Author: Jim Ferenczi +Date: Wed Nov 28 20:15:27 2018 +0100 + + Remove custom QueryBuilder#analyzeGraphPhrase (#35983) + + Now that https://issues.apache.org/jira/browse/LUCENE-8479 is fixed + we can remove the custom implementation of QueryBuilder#analyzeGraphPhrase + in the match QueryBuilder. + +commit 4b85769d24343a31e1280edff18cc303d735de0a +Author: Luca Cavanna +Date: Wed Nov 28 20:06:40 2018 +0100 + + Increase InternalHistogramTests coverage (#36004) + + In `InternalHistogramTests` we were randomizing different values but `minDocCount` was hardcoded to `1`. It's important to test other values, especially `0` as it's the default. To make this possible, the test needed some adapting in the way buckets are randomly generated: all aggs need to share the same `interval`, `minDocCount` and `emptyBucketInfo`. Also assertions need to take into account that more (or less) buckets are expected depending on `minDocCount`. + + This was originated by #35921 and its need to test adding empty buckets as part of the reduce phase. + + Also relates to #26856 as one more key comparison needed to use `Double.compare` to properly handle `NaN` values, which was triggered by the increased test coverage. + +commit 513e1ed095131665fb4aecedac2dea58cb6adce0 +Author: Benjamin Trent +Date: Wed Nov 28 11:43:57 2018 -0600 + + ML: Adding XContentObjectTransformer class (#35957) + + * ML: Adding XContentObjectTransformer class + + * adding license headers + + * Adding custom deprecation handler, and test for checking parsing failures + + * forwarding deprection logs to LoggingDeprecationHandler + +commit a72430e1ef3697f5036bf6b8717135c429327cc4 +Author: Lisa Cawley +Date: Wed Nov 28 09:38:58 2018 -0800 + + [DOCS] Adds TLS warning to rolling upgrades (#35841) + +commit daef498cd96baead4945235ce3cbb04a3d7a45b7 +Author: Simon Willnauer +Date: Wed Nov 28 16:55:29 2018 +0100 + + [DOCS] Fix HLRC reference to unfreeze_index + +commit 89e4ac8fccb1b6a1d907ef467438011fc72dd850 +Author: Simon Willnauer +Date: Wed Nov 28 15:42:12 2018 +0100 + + Add high-level REST client API for `_freeze` and `_unfreeze` (#35723) + + This change adds support for `_freeze` and `_unfreeze` to the HLRC + + Relates to #34352 + +commit 0588dad80b4463ecf571207f18ca684dfc577041 +Author: Nik Everett +Date: Wed Nov 28 09:28:27 2018 -0500 + + Tasks: Only require task permissions (#35667) + + Right now using the `GET /_tasks/` API and causing a task to opt + in to saving its result after being completed requires permissions on + the `.tasks` index. When we built this we thought that that was fine, + but we've since moved towards not leaking details like "persisting task + results after the task is completed is done by saving them into an index + named `.tasks`." A more modern way of doing this would be to save the + tasks into the index "under the hood" and to have APIs to manage the + saved tasks. This is the first step down that road: it drops the + requirement to have permissions to interact with the `.tasks` index when + fetching task statuses and when persisting statuses beyond the lifetime + of the task. + + In particular, this moves the concept of the "origin" of an action into + a more prominent place in the Elasticsearch server. The origin of an + action is ignored by the server, but the security plugin uses the origin + to make requests on behalf of a user in such a way that the user need + not have permissions to perform these actions. It *can* be made to be + fairly precise. More specifically, we can create an internal user just + for the tasks API that just has permission to interact with the `.tasks` + index. This change doesn't do that, instead, it uses the ubiquitus + "xpack" user which has most permissions because it is simpler. Adding + the tasks user is something I'd like to get to in a follow up change. + + Instead, the majority of this change is about moving the "origin" + concept from the security portion of x-pack into the server. This should + allow any code to use the origin. To keep the change managable I've also + opted to deprecate rather than remove the "origin" helpers in the + security code. Removing them is almost entirely mechanical and I'd like + to that in a follow up as well. + + Relates to #35573 + +commit 51a7dc54ece45e774b3383d44ec40a9202b80f00 +Author: Christoph Büscher +Date: Wed Nov 28 15:07:11 2018 +0100 + + Fix custom AUTO issue with Fuzziness#toXContent (#35807) + + Currently when a Fuzziness instance with custom AUTO distance values gets + written to XContent, the customized lower and upper distance values are ommited + and can consequently not be parsed back. This changes this to write the String + including the optional custom values when writing to XContent and fixes the + tests that should have caught this in the first place, e.g. by adding the custom + low and high distance values to the equality check. + +commit 0b45fb98b97e43ddccba7bf511859563a46d5c19 +Author: Andrey Ershov +Date: Wed Nov 28 14:49:07 2018 +0100 + + [Zen2] Generate coordinationMetaData with different configs (#35991) + + This PR fixes test failure, which is caused by equal randomly generated lastAcceptedConfiguration and lastCommittedConfguration. + +commit a2338bb116f40a94375e61b587eeab4ddb57e432 +Author: Marios Trivyzas +Date: Wed Nov 28 14:29:10 2018 +0100 + + SQL: Fix issue with wrong data type for scripted Grouping keys (#35969) + + When the grouping key of a GROUP BY is a painless script (functions are + involved), the data type of the key was incorrect in certain cases + (Boolean, IP, Date). This resulted in returning wrong data type for this + columns in the query results. E.g.: + + ``` + SELECT COUNT(*), a > 10 AS a FROM t GROUP BY a + ``` + + Fixes: #35662 + +commit a3186e4a3289dccfbc253dc8469fb6fc26febffe +Author: Jason Tedor +Date: Wed Nov 28 08:24:35 2018 -0500 + + Deprecate X-Pack centric license endpoints (#35959) + + This commit is part of our plan to deprecate and ultimately remove the + use of _xpack in the REST APIs. + +commit 78ac12d95235b26fec25946af226bbb5e151967d +Author: Jason Tedor +Date: Wed Nov 28 08:24:15 2018 -0500 + + Remove deprecated Graph endpoints (#35956) + + We had some endpoints in Graph that are deprecated for removal in + 7.0.0. This commit removes these deprecated endpoints. + +commit aabff7318efba2037ae4e983706c899cf2efc5a7 +Author: Andrei Stefan +Date: Wed Nov 28 15:20:04 2018 +0200 + + SQL: DATABASE() and USER() system functions (#35946) + +commit 5f0c03618300f89e94a2e8b4fa3c84e0b2f2e433 +Author: Yannick Welsch +Date: Wed Nov 28 13:34:28 2018 +0100 + + Disable testDeleteCreateInOneBulk on Zen2 + + This test needs adaptation to run with Zen2 + +commit 8e86340d1d7747bcc2197914b07a26690c91a63f +Author: Hendrik Muhs +Date: Wed Nov 28 13:34:20 2018 +0100 + + set line lengths for comments equal to code (#35984) + + eclipse (default) wraps code comments at 80 characters, this change sets line length for code comments equal to code == 140. + +commit 1f25a0bd31f86ddc7231e8a302321cd2c4a525e6 +Author: Peter Dyson +Date: Wed Nov 28 21:04:57 2018 +1000 + + [Docs] Add example for updating meta field (#35893) + +commit 33865211dbde64ab4f77de20db4d88a640feeb3b +Author: Christoph Büscher +Date: Wed Nov 28 12:01:27 2018 +0100 + + [Docs] Emphazise suggest behaviour with missing query part (#35393) + + Add a short extra sentence that explains that a missing query part in a search + request containing a "suggest" section will mean only suggestions are returned. + + Closes #31640 + +commit f0c5f0c099f5afebb5cb378de1cbb96c89faa925 +Author: Vladimir Dolzhenko +Date: Wed Nov 28 11:58:20 2018 +0100 + + [HLRC] XPack ML info action (#35777) + + Relates to #29827 + +commit 95346926efc4ca6f67ce4fbe3909860afef5ed54 +Author: Costin Leau +Date: Wed Nov 28 12:48:26 2018 +0200 + + SQL: Lock down JDBC driver (#35798) + + Move classes under the same package to avoid internal classes being + exposed to the outside. Remove public visibility outside 3 classes: + EsDriver, EsDataSource and EsTypes. + The driver only has one package, namely org.elasticsearch.xpack.sql.jdbc + Use Es prefix for classes to ease name conflict and indicate their + destination + + Fix #35437 + +commit 2f547bac6516f88c534263ca3ea894022711fd60 +Author: Christoph Büscher +Date: Wed Nov 28 11:09:03 2018 +0100 + + Remove deprecated methods from QueryStringQueryBuilder (#35912) + + This change removes the deprecated useDisMax() and useAllFields() methods from + the QueryStringQueryBuilder and related tests. The disMax parameter has already + been a no-op since 6.0 and also the useAllFields has been deprecated since 6.0 + and there is a direct replacement via defaultField. + +commit d948edfd695494024dc2506495c0dcaaec750255 +Author: Ioannis Kakavas +Date: Wed Nov 28 11:34:41 2018 +0200 + + [TESTS] Mute SmokeTestWatcherWithSecurityIT tests + + Issue tracked in + https://github.com/elastic/elasticsearch/issues/35361 + +commit fc7e7e7d81da60450d2a03245aaf5f52e4164a66 +Author: Ioannis Kakavas +Date: Wed Nov 28 09:35:36 2018 +0200 + + [TESTS] Mute SNI tests on FIPS 140 JVMs + + These tests use a TrustAllConfig TrustManager as they set + verification_mode to none, that can't be used in a FIPS 140 JVM. + +commit c42d9d91c9074833da5de854911f504a5d634eef +Author: Jason Tedor +Date: Tue Nov 27 22:16:21 2018 -0500 + + Deprecate X-Pack centric SQL endpoints (#35964) + + This commit is part of our plan to deprecate and ultimately remove the + use of _xpack in the REST APIs. + +commit e50e0f997aabdabfe1dca8bf22e850bb0c15205d +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Nov 28 13:29:43 2018 +1100 + + [Kerberos] Add support for Kerberos V5 Oid (#35764) + + Clients can use the Kerberos V5 security mechanism and when it + used this to establish security context it failed to do so as + Elasticsearch server only accepted Spengo mechanism. + This commit adds support to accept Kerberos V5 credentials + over spnego. + + Closes #34763 + +commit 19ed17195fbf2c3ddee7b1563e61cc29340cf12b +Author: Michael Basnight +Date: Tue Nov 27 19:40:49 2018 -0600 + + Docs: DRY up indices docs (#35971) + + This commit DRYs up the indices folder as well as fixing a few minor + mishaps that were in the docs. + +commit b84f1f6a3ad511538ea102badfefa934fa8f0336 +Author: Jason Tedor +Date: Tue Nov 27 20:34:17 2018 -0500 + + Deprecate X-Pack centric rollup endpoints (#35962) + + This commit is part of our plan to deprecate and ultimately remove the + use of _xpack in the REST APIs. + +commit 36819f78ef365480ebc20b2126eeff02a86f1394 +Author: Albert Zaharovits +Date: Wed Nov 28 01:24:03 2018 +0200 + + DOCS Audit event attributes in new format (#35510) + + Accounts for the `Structured Audit Entries` in the format + documentation. + +commit 383713d4e9cfddc9a34e5794b08c447feb33f2a7 +Author: Ryan Ernst +Date: Tue Nov 27 15:18:00 2018 -0800 + + Scripting: Actually add joda time back to whitelist (#35965) + + This commit actually loads the joda whitelist, which was missed in + +commit a0b319a1238d46ca646f9d118482ddeeb511aeba +Author: Tal Levy +Date: Tue Nov 27 14:28:25 2018 -0800 + + [DOCS] fix HLRC ILM doc misreferenced tag + +commit 580b5baf215e7615680b5b3dc85e2cb9d0fa04ba +Author: Ioannis Kakavas +Date: Tue Nov 27 23:35:42 2018 +0200 + + Add realm information for Authenticate API (#35648) + + - Add the authentication realm and lookup realm name and type in the response for the _authenticate API + - The authentication realm is set as the lookup realm too (instead of setting the lookup realm to null or empty ) when no lookup realm is used. + +commit fe603e9163ac6bf03a4709f6ee655c11013a4806 +Author: Tal Levy +Date: Tue Nov 27 13:28:58 2018 -0800 + + [ILM] add HLRC docs to remove-policy-from-index (#35759) + + This primarily introduces documentation for the HLRC + remove-policy-from-index action. + +commit 1a1092d08f27d534db5c579973f4d076b6c48cfb +Author: Zachary Tong +Date: Tue Nov 27 16:02:48 2018 -0500 + + [Rollup] Update serialization version after backport + +commit 48fa251812ccf19da918831a6b8cbad7693a99fa +Author: Zachary Tong +Date: Tue Nov 27 15:46:10 2018 -0500 + + [Rollup] Add more diagnostic stats to job (#35471) + + * [Rollup] Add more diagnostic stats to job + + To help debug future performance issues, this adds the + min/max/avg/count/total latencies (in milliseconds) for search + and bulk phase. This latency is the total service time including + transfer between nodes, not just the `took` time. + + It also adds the count of search/bulk failures encountered during + runtime. This information is also in the log, but a runtime counter + will help expose problems faster + + * review cleanup + + * Remove dead ParseFields + +commit af9233b067e60d646956c2eaacbbd7b9a18c830a +Author: Marios Trivyzas +Date: Tue Nov 27 21:31:43 2018 +0100 + + Build: Fix gradle build for Mac OS (#35968) + + Remove trailing line feed from the output of the command to get + the number of CPUs. + + Follow up to: #35789 + +commit 49087f16f51c79bd3f50a61d5214522cbf122d35 +Author: Jeff Hajewski +Date: Tue Nov 27 13:30:13 2018 -0600 + + Adds deprecation logging to ScriptDocValues#getValues. (#34279) + + `ScriptDocValues#getValues` was added for backwards compatibility but no + longer needed. Scripts using the syntax `doc['foo'].values` when + `doc['foo']` is a list should be using `doc['foo']` instead. + + Closes #22919 + +commit 3337fa73511151453580a6ec181efdb3b62de080 +Author: Chris Earle +Date: Tue Nov 27 14:28:14 2018 -0500 + + [Monitoring] Make Exporters Async (#35765) + + This changes the exporter code -- most notably the `http` exporter -- + to use async operations throughout the resource management and bulk + initialization code (the bulk indexing of monitoring documents was + already async). + + As part of this change, this does change one semi-core aspect of the + `HttpResource` class in that it will no longer block all concurrent calls + until the first call completes with + `HttpResource::checkAndPublishIfDirty`. + Now, any parallel attempts to check the resources will be skipped until + the first call completes (success or failure). While this is a technical + change, it has very little practical impact because the existing behavior + was either quick success (then every blocked request processed) or + each request timed out and failed anyway, thus being effectively + skipped (and a burden on the system). + +commit c2329cdf1dd89cf7cb04413b211bd19f11ec130a +Author: Tal Levy +Date: Tue Nov 27 11:03:47 2018 -0800 + + [ILM] reduce time restriction on IndexLifecycleExplainResponse (#35954) + + step times were set. The assumption was that these are always set. + Tests passed, which led me to believe this was true. There is a time + when shrunk indices have their step phase/action/step details set, + but with no time information (in the CopyExecutionStateStep). + Explain API fails for these + +commit 2061eeb1226168444f664bcbb00d4efa0f70dd09 +Author: Jay Modi +Date: Tue Nov 27 11:28:26 2018 -0700 + + Remove use of AbstractComponent in xpack (#35394) + + This commit removes the use of AbstractComponent in xpack where it was + still being extended. It has been replaced with explicit logger + declarations. + + See #34488 + +commit 25c416b12d957990f0b7a7b98ff73cbc7dd045db +Author: Julie Tibshirani +Date: Tue Nov 27 10:19:19 2018 -0800 + + Deprecate types in search and multi search templates. (#35669) + + This PR adds deprecation warnings to the relevant `Rest*Action` classes, plus tests in `Rest*ActionTests`. No updates to REST tests, the Java HLRC, or documentation were necessary, since they didn't make use of types. + +commit 7e772e311d70949b90bd243b1a91b42b64ec74af +Author: Luca Cavanna +Date: Tue Nov 27 19:04:44 2018 +0100 + + Remove fromXContent from IndexUpgradeInfoResponse (#35934) + + Such method is there only for testing purposes, it is not needed. + +commit ad1f0dccd47f20a403adc6cc2447ab1c3d39b558 +Author: Simon Willnauer +Date: Tue Nov 27 17:08:24 2018 +0100 + + Validate metdata on `_msearch` (#35938) + + MultiSearchRequests issues through `_msearch` now validate all keys + in the metadata section. Previously unknown keys were ignored + while now an exception is thrown. + + Closes #35869 + +commit cc1fa799c821d7c7b2a22c0f23c0fdaafbcd5cff +Author: Tim Brooks +Date: Tue Nov 27 09:08:14 2018 -0700 + + Remove `TcpChannel#setSoLinger` method (#35924) + + This commit removes the dedicated `setSoLinger` method. This simplifies + the `TcpChannel` interface. This method has very little effect as the + SO_LINGER is not set prior to the channels being closed in the abstract + transport test case. We still will set SO_LINGER on the + `MockNioTransport`. However we can do this manually. + +commit b6ed6ef1892150aadbb1eb330f88bc71628b345e +Author: Tim Brooks +Date: Tue Nov 27 09:06:52 2018 -0700 + + Add sni name to SSLEngine in nio transport (#35920) + + This commit is related to #32517. It allows an "sni_server_name" + attribute on a DiscoveryNode to be propagated to the server using + the TLS SNI extentsion. Prior to this commit, this functionality + was only support for the netty transport. This commit adds this + functionality to the security nio transport. + +commit adc0b560c071049fba8b934951f1ba63d41904c6 +Author: Christophe Bismuth +Date: Tue Nov 27 16:35:45 2018 +0100 + + Raise a 404 exception when document source is not found (#33384) (#34083) + + This pull request makes the `RestGetSourceAction` return a `ResourceNotFoundException` with a proper JSON response when source or document itself is missing (see issue #33384). + + Here is below a sample JSON output: + + ``` + { + "error": { + "root_cause": [ + { + "type": "resource_not_found_exception", + "reason": "Source not found [index1]/[_doc]/[1]" + } + ], + "type": "resource_not_found_exception", + "reason": "Source not found [index1]/[_doc]/[1]" + }, + "status": 404 + } + ``` + +commit df341aba4a44e070430d56491035d0761d3ad373 +Author: Tal Levy +Date: Tue Nov 27 07:29:59 2018 -0800 + + [ILM] make IndexLifecycleExplainResponse more resilient to null values (#35800) + + added validation for complete information of step details. + + also changed the rendering of explain responses so null strings are not rendered + + Another thing that I changed is the format of the client-side response. I found it difficult to maintain the two subtly-different objects, so I migrated the usage of long for the fields, to Long (just as it is on the server-side). + +commit a615ab454a5501cd5f6ef9d4b3e24d7e4b715d06 +Author: Alexander Reelsen +Date: Tue Nov 27 16:15:37 2018 +0100 + + Watcher: Only trigger a watch if new or schedule/changed (#35908) + + The trigger engine did always create a new schedule data structure, when + the watcher indexing listener called an add. However the indexing + listener also called add, when the watch status was updated. This means, + that upon a watch status update the watch got retriggered, potentially + waiting a defined interval from the watch status update onwards, instead + of waiting from the last run. + + This commit only updates the schedule in the trigger engine, if it + actually has changed, otherwise the existing schedule will not be + touched. This has two results + + 1. If a watch is updated by an execution, the existing interval will not + be touched (meaning the scheduled time will not move forward). + 2. If a watch is updated by a user, but the schedule is not changed, it + will not be reset from the update (for example starting to count from 5 + minutes again, if the interval was set to 5 minutes). + + Furthermore some minor cleanups were applied, making variables final in + the ctor, preventing double creation of variables. + +commit 38725bd3a1b4a9871ff73825fa3a74382adce885 +Author: Nik Everett +Date: Tue Nov 27 09:59:44 2018 -0500 + + Build: Pick default test jvms for macOS (#35789) + + In #35259 we switched the default number of VMs to fork for unit tests to + the number of physical CPU cores. But because we could only get an accurate + count on machines with a normal `/proc` filesystem, macOS machine did not + pick up the new default. Given that macOS is a huge portion of developer + machines, we'd like to get the right default there. This does that. + + It also moves the default-finding process from happening once per testing + task to happening once at startup. This seems like a good choice in general, + but a very good choice for macOS because we have to run a command to list + the count. + +commit 541869a96ad234944b9aff3123a4bf7ff5e1d616 +Author: Alpar Torok +Date: Tue Nov 27 16:38:56 2018 +0200 + + Add missing entries to conffiles (#35810) + +commit 1da9c6faa07820e77318e9987d89edfd399a92fe +Author: Marios Trivyzas +Date: Tue Nov 27 15:33:43 2018 +0100 + + SQL: Fix translation of math functions to painless (#35910) + + `SIGN` and `RADIANS` where wrongly overriding `mathFunction()`. + Converted `mathFunction()` to private in `MathFunction` since it + shouldn't be overriden, as it uses the assigned `MathOperation` + to get the funtion name for painless scripts. + + Fixes: #35654 + +commit 8d9e21ffaaab9a7ef1b6d5b7bf69c29f0728a6e8 +Author: Tim Vernum +Date: Wed Nov 28 01:10:14 2018 +1100 + + Fix mutator in GetRolesRequestTests (#35932) + + There was no guarantee that the mutated object would not be the same + as the original object (and was for seed 6A8C4CBF63B5AA63) + + Resolves #35947 + +commit cb8646ac0e0509ef609638a049342b78318b4105 +Author: Nik Everett +Date: Tue Nov 27 09:06:10 2018 -0500 + + Tests: If dpkg fails try and log failure (#35551) + + If `dpkg` fails, try and look for who has `/var/lib/dpkg/lock` open. If + it exists and is open then return a failure with information about who + has file open. This should help us debug #33762. + + Closes #34309 + +commit 0e283f967060e3f3c65aac9331680abb85277292 +Author: Andrey Ershov +Date: Tue Nov 27 15:04:52 2018 +0100 + + [Zen2] PersistedState interface implementation (#35819) + + Today GatewayMetaState is capable of atomically storing MetaData to + disk. We've also moved fields that are needed to be persisted in Zen2 + from ClusterState to ClusterState.MetaData.CoordinationMetaData. + + This commit implements PersistedState interface. + + version and currentTerm are persisted as a part of Manifest. + GatewayMetaState now implements both ClusterStateApplier and + PersistedState interfaces. We started with two descendants + Zen1GatewayMetaState and Zen2GatewayMetaState, but it turned + out to be not easy to glue it. + GatewayMetaState now constructs previousClusterState (including + MetaData) and previousManifest inside the constructor so that all + PersistedState methods are usable as soon as GatewayMetaState + instance is constructed. Also, loadMetaData is renamed to + getMetaData, because it just returns + previousClusterState.metaData(). + Sadly, we don't have access to localNode (obtained from + TransportService in the constructor, so getLastAcceptedState + should be called, after setLocalNode method is invoked. + Currently, when deciding whether to write IndexMetaData to disk, + we're comparing current IndexMetaData version and received + IndexMetaData version. This is not safe in Zen2 if the term has changed. + So updateClusterState now accepts incremental write + method parameter. When it's set to false, we always write + IndexMetaData to disk. + Things that are not covered by GatewayMetaStateTests are covered + by GatewayMetaStatePersistedStateTests. + This commit also adds an option to use GatewayMetaState instead of + InMemoryPersistedState in TestZenDiscovery. However, by default + InMemoryPersistedState is used and only one test in PersistedStateIT + used GatewayMetaState. In order to use it for other tests, proper + state recovery should be implemented. + +commit c91ef1105dd03e1538e80bc329c3509a11a36a1a +Author: Marios Trivyzas +Date: Tue Nov 27 14:46:11 2018 +0100 + + SQL: Implement data type verification for conditionals (#35916) + + Add special verifier rule to check that the arguments of conditional + functions are of the same or compatible types. This way the user gets + a descriptive error message with line number and column indicating + where is the offending argument. + + Closes: #35907 + +commit 110c4fdd650e14d593bd66c2c46e55349c23c501 +Author: David Roberts +Date: Tue Nov 27 12:52:52 2018 +0000 + + [ML] Adjust file structure finder parser config (#35935) + +commit b6eb73a835a94d55f66646c082d0c9ec81a28944 +Author: Tim Vernum +Date: Tue Nov 27 21:05:12 2018 +1100 + + Update BWC checks in HasPrivilegesResponse (#35929) + + This serialization has been backported to 6.x + +commit 4a3d66c668093178d5501fc4b740b56c35a4f512 +Author: Martijn van Groningen +Date: Tue Nov 27 10:52:53 2018 +0100 + + [TEST] wrong bugUrl... + + Relates to #35937 + +commit fabffb89b8980d6d684b8569b4c52be7a67dd0af +Author: Martijn van Groningen +Date: Tue Nov 27 10:49:03 2018 +0100 + + Muted test. This test expose an issue inside the auto follower coordinator, + it has nothing to do with the hlrc support for put auto follow pattern api, + this test was added for. + + Relates to #35480 + +commit 31fdb7697370d1dba4777a0156faee02ed4578cd +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Nov 27 19:09:17 2018 +1100 + + [DOCS] Document authorization_realms for Kerberos realm (#35927) + + This commit adds documentation for authorization_realms + setting for the Kerberos realm and also corrects a typo in + existing documentation. + + Co-authored-by: @A-Hall + +commit df1e02d0d55f0981212e7d33f5e52e35a02d6fcc +Author: Martijn van Groningen +Date: Tue Nov 27 08:53:22 2018 +0100 + + [HLRC] Added support for CCR Put Auto Follow Pattern API (#35780) + + This change also adds documentation for the Put Auto Follow Pattern API. + + Relates to #33824 + +commit 447e5d212a2c1ef49b233b061537986edd9f2f8a +Author: Martijn van Groningen +Date: Tue Nov 27 08:00:06 2018 +0100 + + Changed versions in serialization code after backporting #35535 + +commit 998c663f0d497ba6fe5d4e848ac192bcb90b491a +Author: Ioannis Kakavas +Date: Tue Nov 27 08:18:18 2018 +0200 + + Add Tests for findSamlRealm (#35905) + + This commit adds a test for handling correctly all they possible + `SamlPrepareAuthenticationRequest` parameter combinations that + we might get from Kibana or a custom web application talking to the + SAML APIs. + We can match the correct SAML realm based either on the realm name + or the ACS URL. If both are included in the request then both need to + match the realm configuration. + +commit 5b427d415e5de7bc326caa3df6e317e82a60855a +Author: Tim Vernum +Date: Tue Nov 27 15:19:47 2018 +1100 + + Add "request.id" to file audit logs (#35536) + + This generates a synthesized "id" for each incoming request that is + included in the audit logs (file only). + This id can be used to correlate events for the same request (e.g. + authentication success with access granted). + + This request.id is specific to the audit logs and is not used for any + other purpose + + The request.id is consistent across nodes if a single request requires + execution on multiple nodes (e.g. search acros multiple shards). + +commit a18b219f7934decc6607531ebe8dcfcd006fa19b +Author: Tim Vernum +Date: Tue Nov 27 15:08:53 2018 +1100 + + Allow noop PutUser updates (#35843) + + When assertions are enabled, a Put User action that have no effect (a + noop update) would trigger an assertion failure and shutdown the node. + + This change accepts "noop" as an update result, and adds more + diagnostics to the assertion failure message. + +commit 3435fc4613653a159fb8e0353204acfd98a225ca +Author: Tim Vernum +Date: Tue Nov 27 15:07:24 2018 +1100 + + HLRC: Add ability to put user with a password hash (#35844) + + Update PutUserRequest to support password_hash (see: #35242) + + This also updates the documentation to bring it in line with our more + recent approach to HLRC docs. + +commit 72d7f236a5deb6e52226518e5e6889dc1ce64fa1 +Author: Ryan Ernst +Date: Mon Nov 26 16:33:05 2018 -0800 + + Scripting: Add back joda to whitelist (#35915) + + Watcher still exposes some dates as joda DateTime objects. This commit + adds back joda to the painless whitelist so they can still be accessed. + + closes #35913 + +commit 10447dd9624659537a040aef47f2b8605bb08c78 +Author: Ryan Ernst +Date: Mon Nov 26 16:18:35 2018 -0800 + + Build: Fix jdbc jar to include deps (#35602) + + This commit adds back bundling of all deps of the sql jdbc jar. This was + lost in a refactoring of how the shadow plugin is handled for the entire + elasticsearch project. + +commit 119835decd4326168599f835b4a8ac301e2949cc +Author: Gordon Brown +Date: Mon Nov 26 17:05:12 2018 -0700 + + Always enforce cluster-wide shard limit (#34892) + + This removes the option to run a cluster without enforcing the + cluster-wide shard limit, making strict enforcement the default and only + behavior. The limit can still be adjusted as desired using the cluster + settings API. + +commit 4d525e3e330b3182c2fa3d0905403ea1cc311dc1 +Author: Christoph Büscher +Date: Tue Nov 27 00:05:15 2018 +0100 + + [Docs] Align callouts in search.asciidoc (#35897) + +commit 5c11b8612a0528b54a2bff118a0764874a312b00 +Author: Gordon Brown +Date: Mon Nov 26 15:25:56 2018 -0700 + + Add HLRC docs for Explain Lifecycle (#35803) + + Adds HLRC documentation for the Explain Lifecycle API. + +commit 5f656e4eab8781e7ca5a4d3fa0861d6e742f32de +Author: Sachin Araballi +Date: Tue Nov 27 02:35:28 2018 +0530 + + Fields are declared at the top of the class. (#35891) + + Fields should be declared at the top of the class, before any method declarations, constructors, initializers or inner classes. + +commit 8fe5f66637ba0da051b088f53f89beddd0d3f6e6 +Author: Tal Levy +Date: Mon Nov 26 12:26:36 2018 -0800 + + [ILM] fix rest-api-spec doc links (#35478) + +commit 663563f64b7aef4e724bc98dca114f190728b4a4 +Author: Igor Motov +Date: Mon Nov 26 09:44:42 2018 -1000 + + Geo: better handling of malformed geo_points (#35554) + + Improves handling of malformed geo_points when `ignore_malformed` is + set to true + + Closes #35419 + +commit 900caa20efbd370ba788e8fc0292e391a8352f82 +Author: Jim Ferenczi +Date: Mon Nov 26 19:08:14 2018 +0100 + + Handles exists query in composite aggs (#35758) + + This commit adds the support for exists query in the sorted execution mode + of the `composite` aggregation. We'll execute the aggregation from the sorted + points and use early termination if the main query is an `exists` query over the + first source of the `composite` aggregation. + +commit 3f7cae3f0d6ec351def2c66e7a8fea20409c9ede +Author: Marios Trivyzas +Date: Mon Nov 26 18:21:36 2018 +0100 + + SQL: Implement GREATEST and LEAST functions (#35879) + + Add GREATEST(expr1, expr2, ... exprN) and LEAST(expr1, expr2, exprN) + functions which are in the family of CONDITIONAL functions. + + Implementation follows PostgreSQL behaviour, so the functions return + `NULL` when all of their arguments evaluate to `NULL`. + + Renamed `CoalescePipe` and `CoalesceProcessor` to `ConditionalPipe` and + `ConditionalProcessor` respectively, to be able to reuse them for + `Greatest` and `Least` evaluations. To achieve that `ConditionalOperation` + has been added to differentiate between the functionalities at execution + time. + + Closes: #35878 + +commit b95a4db6e61b44a8cb1d7985257cf532165827c7 +Author: Christophe Bismuth +Date: Mon Nov 26 18:15:59 2018 +0100 + + Throw a parsing exception when boost is set in span_or query (#28390) (#34112) + +commit 0baffda390fdbe84239aae25ee433f5b886c1d6c +Author: John +Date: Mon Nov 26 12:13:46 2018 -0500 + + ingest: grok remove duplicated patterns (#35886) + + This commit removes the redundant (and incorrect) JAVACLASS + and JAVAFILE grok patterns. This helps to keep parity with + Logstash's patterns. + + See also: https://github.com/logstash-plugins/logstash-patterns-core/pull/237 + + closes #35699 + +commit 4b99a663c1605e81b267654ab8b2bd5e4d2acffa +Author: Jake Landis +Date: Mon Nov 26 10:34:42 2018 -0600 + + ingest: fix broken doc link + +commit 13e11966cab7a103135fe75a20f7d170dff1b499 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Mon Nov 26 16:15:54 2018 +0000 + + [HLRC][ML] Add delete expired data API (#35906) + + + Relates to #29827 + +commit 3ed3aab12762c4b32eea1a896f7cbcdedf7e13ae +Author: Jim Ferenczi +Date: Mon Nov 26 16:45:56 2018 +0100 + + Fix rest test when numShards is greater than 1 (#35909) + +commit 7f7b31723e363ca3364b1e85c64c1afae6080f8b +Author: Jake Landis +Date: Mon Nov 26 09:35:45 2018 -0600 + + ingest: extended `if` documentation (#35044) + + part of #33188 + +commit d291b080de76e29d0e5873b288bab184fbe3bb60 +Author: Costin Leau +Date: Mon Nov 26 16:59:09 2018 +0200 + + SQL: SYS COLUMNS returns ODBC specific schema (#35870) + + Due to some unresolvable type conflict between the expected definition + in JDBC vs ODBC, the driver mode is now passed over so that certain + command can change their results accordingly (in this case SYS COLUMNS) + + Fix #35376 + +commit ca9b2b993167ed6fa6e71fe9d04ae1050508ce18 +Author: Simon Willnauer +Date: Mon Nov 26 14:26:39 2018 +0100 + + Repsect indices options on _msearch (#35887) + + Today we don't respect the indices options when they are passed + as request parameters to the `_msearch` endpoint. This is unintuitive + and doesn't cause any errors. This changes uses the top-level indices + options as the defaults for each sub search-request. + + Closes #35851 + +commit b078e2970cfb79c040d42e518413e0c325adde43 +Author: Marios Trivyzas +Date: Mon Nov 26 14:02:02 2018 +0100 + + SQL: Implement null safe equality operator `<=>` (#35873) + + This operator handles nulls in different way than the normal `=`. + If one of the operants is `null` and the other not it returns `false`. + If both operants are `null` it returns `true`. Therefore in contrary to + `=`, which returns `null` if at least one of the operants is `null`, this one + never returns `null` as a result. + + Closes: #35871 + +commit 04ebc63e340301f4d4c92c2475b448a55292a983 +Author: Christophe Bismuth +Date: Mon Nov 26 13:57:57 2018 +0100 + + RoutingMissingException in more like this (#33974) + + More like this query allows to provide identifiers of documents to be retrieved as like/unlike items. + It can happen that at retrieval time an error is thrown, for instance caused by missing routing value when `_routing` is set required in the mapping. + Instead of ignoring such error and returning no documents for the query, the error should be re-thrown and returned to users. As part of this + change also mget and mtermvectors are unified in the way they throw such exception like it happens in other places, so that a `RoutingMissingException` is raised. + + Closes #29678 + +commit 9bdbba23f8ca907efc1114cbd390e6263493ca92 +Author: Tanguy Leroux +Date: Mon Nov 26 13:43:46 2018 +0100 + + [Tests] Fix IndexShardTests.testAcquirePrimaryAllOperationsPermits() + + This test fails on CI because of an inappropriate assertion, which is + I think a leftover and has no real value. + +commit e44390ac204ece3e1e770e9aedbac601d6787f51 +Author: Luca Cavanna +Date: Mon Nov 26 13:42:41 2018 +0100 + + InitialSearchPhase minor cleanups (#35864) + + This commit simplifies the throttling logic in InitialSearchPhase and removes some asserts from it. Also, a few formatting changes are applied to its code and surrounding classes. + +commit a68a46450bd040a46d9a947c073debad1c9cd895 +Author: David Turner +Date: Mon Nov 26 10:52:49 2018 +0000 + + [Zen2] Add lag detector (#35685) + + A publication can succeed and complete before all nodes have applied the + published state and acknowledged it, thanks to the publication timeout; however + we need every node eventually either to apply the published state (or a later + state) or be removed from the cluster. This change introduces the LagDetector + which achieves this liveness property by removing any lagging nodes from the + cluster. + +commit cd822b7ca828a14269921757e386944a116d5bc2 +Author: Costin Leau +Date: Mon Nov 26 12:41:43 2018 +0200 + + SQL: Fix incorrect merging of #35852 + +commit 633a562208b23d31c9c998f678be3c8e79c7e663 +Author: Costin Leau +Date: Mon Nov 26 12:35:46 2018 +0200 + + SQL: Fix incorrect merging of #35852 + +commit acb65dd004f43e2052ad805b5fb4052baf041e04 +Author: Ioannis Kakavas +Date: Mon Nov 26 11:45:03 2018 +0200 + + [DOCS] Fix missing include for get-roles + +commit 971299baf4b258dc19c45354417a140a6e9f7586 +Author: Costin Leau +Date: Mon Nov 26 11:26:24 2018 +0200 + + SQL: Add filtering to SYS TYPES (#35852) + + Fix #35342 + +commit 8daa854f90b982854ce13a5a61b0fe29eddf40b5 +Author: Ioannis Kakavas +Date: Mon Nov 26 11:25:07 2018 +0200 + + [HLRC] Add support for get roles API (#35787) + + This commits adds support for the Get Roles API to the HLRC + + Relates: #29827 + +commit dfd93deabef3f5647e268c397e28a737622262bd +Author: Yannick Welsch +Date: Mon Nov 26 09:57:40 2018 +0100 + + Remove superfluous comment + +commit 401b814d1acf56870bbd11d46ec81e07ac692a3f +Author: iverase +Date: Mon Nov 26 09:33:23 2018 +0100 + + [CI] Muting method testOperationPermitOnReplicaShards in IndexShardTests + + Relates to #35850 + +commit 7624734f14bc77acc299a448b108113a8694e5a3 +Author: Martijn van Groningen +Date: Mon Nov 26 08:50:08 2018 +0100 + + Added wait_for_metadata_version parameter to cluster state api. (#35535) + + The `wait_for_metadata_version` parameter will instruct the cluster state + api to only return a cluster state until the metadata's version is equal or + greater than the version specified in `wait_for_metadata_version`. If + the specified `wait_for_timeout` has expired then a timed out response + is returned. (a response with no cluster state and wait for timed out flag set to true) + In the case metadata's version is equal or higher than `wait_for_metadata_version` + then the api will immediately return. + + This feature is useful to avoid external components from constantly + polling the cluster state to whether somethings have changed in the + cluster state's metadata. + +commit 00e6fec718b7e1ae9c3c9ec2968a5af4b51870b9 +Author: Andrei Stefan +Date: Mon Nov 26 07:11:43 2018 +0200 + + SQL: XPack FeatureSet functionality (#35725) + + * Introduced "client.id" parameter for REST requests + * Bug that made the Verifier run twice, fixed in the Analyzer + * Single node IT and unit testing + +commit a0f8a22d539b6acf077cc661f24ec4b5fa3541f7 +Author: Tal Levy +Date: Sun Nov 25 17:19:17 2018 -0800 + + [TEST] ILM skip checking _ilm/explain response (#35830) + + relates #35562 + + Closes #35824. + +commit 04a48430aa59fde2855a522db90e9d8f5899c56e +Author: Armin Braun +Date: Sun Nov 25 10:17:54 2018 +0100 + + NETWORKING: Upgrade Netty 4.3.31.Final (#35877) + +commit 05620ccaafe5b8357d8729c9e1101391bb95732a +Author: Armin Braun +Date: Sat Nov 24 19:24:59 2018 +0100 + + TESTS: Disable Snapshot Repo Verify in BwC Test (#35876) + + * Disable snapshot repo verification temporarily because it runs into HTTP-500 in 6.6.0-SNAPSHOT + * Relates #35874 + +commit 4711c5cdf35dcee963ce210e2d2da15469c755ec +Author: Simon Willnauer +Date: Sat Nov 24 09:25:42 2018 +0100 + + Always return false from `refreshNeeded` on ReadOnlyEngine (#35837) + + Acquiring a searcher is unnecessary to determine if a refresh is + necessary since read-only engines never refresh. + + Closes #35785 + +commit e46e44ce3881263785e5f8467b078c80e80cbf50 +Author: Simon Willnauer +Date: Sat Nov 24 09:23:53 2018 +0100 + + Wrap can_match reader with ElasticsearchDirectoryReader (#35857) + + Code that operates on-top of the engine requires all readers returned to be + unwrapped into ElasticsearchDirectoryReader. The special reader + the FrozenEngine uses wasn't wrapped. + +commit 410f570d5f02bd8ed08d6c06a93f4cabd3c78d3d +Author: Marios Trivyzas +Date: Fri Nov 23 22:19:27 2018 +0100 + + SQL: Implement NULLIF(expr1, expr2) function (#35826) + + NULLIF returns null if the 2 expressions are equal or the + expr1 otherwise. + + Closes: #35818 + +commit f47636b2549222f23769ae6ac37261bd8cebb1d8 +Author: Andrey Ershov +Date: Fri Nov 23 18:34:06 2018 +0100 + + [Zen2] Introduce VotingTombstone class (#35832) + + Today voting tombstones are stored in CoordinationMetaData as + Set. + DiscoveryNode is not a lightweight object and have a lot of fields. + It also has toXContent method, but no fromXContent method and the + output of toXContent is not enough to re-create DiscoveryNode + object. + And votingTombstone set should be persisted as a part of MetaData. + On the other hand, the only thing required from the tombstone is the + nodeId. + This PR adds VotingTombstone class for voting tombstones, which + consists of two fields for now - nodeId and nodeName. It could be + extended/shrank in the future if needed. + This PR also resolves TODO's related to the voting tombstones xcontent + story. + Example of CoordinationMetaData.toXContent with voting tombstones: + + { + "term": 1, + "last_committed_config": [ + "fkwLdOBvXSlgRTBfgNAL", + "tmQiPGHvUxXzPkkCDSJo", + "HhOmtQBZAThpHIGWhxpz", + "qZHWGpoDNPYRNIiqKsDl" + ], + "last_accepted_config": [ + "lhqacKmriwhHGFZcvqbx", + "MYysmBuROkvJRlDcusyd" + ], + "voting_tombstones": [ + { + "node_id": "McjbZbRkEz", + "node_name": "pdKIWeNJUO" + }, + { + "node_id": "cpXkVibGwo", + "node_name": "UnCvFgdVsc" + }, + { + "node_id": "EylRNOztbc", + "node_name": "ohOhkbMWZX" + } + ] + } + +commit 51d2e986c5724b1e58a679985513891981954ac5 +Author: Yannick Welsch +Date: Fri Nov 23 17:11:06 2018 +0100 + + Remove BWC conditions after backport of #35731 + + This PR was backported to 6.x, so the extra BWC conditions are not needed anymore + +commit e22a83554eb0bba81e3f7e1b3f353a243fac0c7b +Author: Costin Leau +Date: Fri Nov 23 17:14:52 2018 +0200 + + SQL: Polish grammar for intervals (#35853) + +commit a204d1cdff3503d23e4e1f04ed1fc67cd919378a +Author: William Desportes +Date: Fri Nov 23 15:13:44 2018 +0100 + + [Docs] Fix typo in datehistogram-aggregation.asciidoc (#35855) + +commit 5b370316a6d07f93ab2d9c3918befc4113e65980 +Author: Adrien Grand +Date: Fri Nov 23 15:15:26 2018 +0100 + + Remove some legacy code from when indices could have multiple types. (#35815) + + This code is only necessary up to indices created with version 5.x while 7.0 + only supports indices created with 6.x or 7.0. + +commit 2970abfce91a3f1938536179e6cd18280df39624 +Author: Yannick Welsch +Date: Fri Nov 23 14:45:05 2018 +0100 + + Add read-only repository verification (#35731) + + Adds a verification mode for read-only repositories. It also makes the extra bucket check on + repository creation obsolete, which fixes #35703. + +commit 88d862e69ff2406ad7eda41bd181e3080480dd79 +Author: Christoph Büscher +Date: Fri Nov 23 14:28:02 2018 +0100 + + [CI] Muting two methods in IndexShardTests + + Relates to #35850 + +commit d01436de3c55b9c206b89b868c5281a68b06f7d4 +Author: David Turner +Date: Fri Nov 23 08:43:34 2018 +0000 + + Copy checkpoint atomically when rolling generation (#35407) + + Today when rolling a transog generation we copy the checkpoint from + `translog.ckp` to `translog-nnnn.ckp` using a simple `Files.copy()` followed by + appropriate `fsync()` calls. The copy operation is not atomic, so if we crash + at the wrong moment we can leave an incomplete checkpoint file on disk. In + practice the checkpoint is so small that it's either empty or fully written. + However, we do not correctly handle the case where it's empty when the node + restarts. + + In contrast, in `recoverFromFiles()` we _do_ copy the checkpoint atomically. + This commit extracts the atomic copy operation from `recoverFromFiles()` and + re-uses it in `rollGeneration()`. + +commit be69a774dffa7be3335aed261e3091e417336c39 +Author: Jim Ferenczi +Date: Fri Nov 23 09:42:23 2018 +0100 + + Fix analyzed prefix query in query_string (#35756) + + This change fixes analyzed prefix queries in `query_string` to be ignored + if all terms are removed during the analysis. + + Closes #31702 + +commit 2e37f17a7d70b0f14532fdcd955a4f9ed9ad5bed +Author: Tanguy Leroux +Date: Fri Nov 23 09:26:38 2018 +0100 + + Expose all permits acquisition in IndexShard and TransportReplicationAction (#35540) + + This pull request exposes two new methods in the IndexShard and + TransportReplicationAction classes in order to allow transport replication + actions to acquire all index shard operation permits for their execution. + + It first adds the acquireAllPrimaryOperationPermits() and the + acquireAllReplicaOperationsPermits() methods to the IndexShard class + which allow to acquire all operations permits on a shard while exposing + a Releasable. It also refactors the TransportReplicationAction class to + expose two protected methods (acquirePrimaryOperationPermit() and + acquireReplicaOperationPermit()) that can be overridden when a transport + replication action requires the acquisition of all permits on primary and/or + replica shard during execution. + + Finally, it adds a TransportReplicationAllPermitsAcquisitionTests which + illustrates how a transport replication action can grab all permits before + adding a cluster block in the cluster state, making subsequent operations + that requires a single permit to fail). + + Related to elastic #33888 + +commit 3c059ee057bfd15a120defb6fbdd9343a83b9ad5 +Author: David Roberts +Date: Fri Nov 23 06:58:05 2018 +0000 + + [HLRC][ML] Add ML find file structure API (#35833) + + Relates to #29827 + +commit 813e053f62bb067d12a8a8965fe31ca82d938a27 +Author: Martijn van Groningen +Date: Fri Nov 23 07:52:29 2018 +0100 + + Fixed response classes in hlrc docs + +commit fb80d298de922357ef21ac5e2f37eae27fcc613d +Author: Stéphane Campinas +Date: Fri Nov 23 05:29:20 2018 +0100 + + [Docs] Correct template example description #35829 + +commit 6fb7ce66e6cd9680b0b391bf3d8bf7d837373631 +Author: Christoph Büscher +Date: Fri Nov 23 05:03:19 2018 +0100 + + [Tests] Fix creating ExplainLifecycleRequest with no indices (#35828) + + We didn't check that the ExplainLifecycleRequest was constructed with at least + one index before, now that we do we must also make sure the tests + mutateInstance() method used in equals/hashCode checks doesn't accidentally + create an empty index array. + + Closes #35822 + +commit 6f46584380361d4fa52feebe404cf78a3074bf68 +Author: David Roberts +Date: Thu Nov 22 16:23:31 2018 +0000 + + [ML] Add docs for ML info endpoint (#35783) + + This endpoint was not previously documented as it was not + particularly useful to end users. However, since the HLRC + will support the endpoint we need some documentation to + link to. + + The purpose of the endpoint is to provide defaults and + limits used by ML. These are needed to fully understand + configurations that have missing values because the missing + value means the default should be used. + + Relates #35777 + +commit 43d6ec8bcd7a15aa81e38cc6ed7d651d7a77fb73 +Author: Dimitris Athanasiou +Date: Thu Nov 22 15:08:46 2018 +0000 + + Remove unnecessary throws IOException in CompressedXContent.string() (#35821) + +commit e37a0ef84425ac7497ae4de9439220811fccdfc9 +Author: Jim Ferenczi +Date: Thu Nov 22 15:42:59 2018 +0100 + + Upgrade to lucene-8.0.0-snapshot-67cdd21996 (#35816) + +commit 96a741f2c2834e9a19a15e649e09254017e43c7e +Author: Martijn van Groningen +Date: Thu Nov 22 14:52:59 2018 +0100 + + [TEST] escape brackets + + Relates to #35496 + +commit 4fc911a1294ecbe5db51e8eb1f177a8cf04079d7 +Author: Albert Zaharovits +Date: Thu Nov 22 15:34:53 2018 +0200 + + Mute test InternalEngineTests + + Relates #35823 + +commit 7daccaa42287688e4251a931381a19286ae43c99 +Author: Albert Zaharovits +Date: Thu Nov 22 15:12:16 2018 +0200 + + Mute test + + Relates #35822 + +commit f9f7261d608e55d6d700134632b52ff5f52e9ac2 +Author: Tanguy Leroux +Date: Thu Nov 22 12:13:09 2018 +0100 + + Revert "Revert "[RCI] Check blocks while having index shard permit in TransportReplicationAction (#35332)"" + + This reverts commit d3d7c01 + +commit b6014d971c408145c0bca78117a5bbee609052f7 +Author: Mayya Sharipova +Date: Thu Nov 22 06:08:48 2018 -0500 + + Forbid negative scores in functon_score query (#35709) + + * Forbid negative scores in functon_score query + + - Throw an exception when scores are negative in field_value_factor + function + - Throw an exception when scores are negative in script_score + function + + Relates to #33309 + +commit 92acf47c168897676016c2eea6a4d7a4e207408f +Author: Marios Trivyzas +Date: Thu Nov 22 11:41:00 2018 +0100 + + SQL: Implement NVL(expr1, expr2) (#35794) + + Add NVL as alias to IFNULL as they have the same + behaviour. Add basic tests and docs. + + Closes: #35782 + +commit 11052b75c7d2578224719dc155ae99ae2649e3c3 +Author: Tanguy Leroux +Date: Thu Nov 22 10:50:12 2018 +0100 + + TransportResyncReplicationAction should not honour blocks (#35795) + + After #35332 has been merged, we noticed some test failures like #35597 + in which one or more replica shards failed to be promoted as primaries + because the primary replica re-synchronization never succeed. + + After some digging it appeared that the execution of the resync action was + blocked because of the presence of a global cluster block in the cluster state + (in this case, the "no master" block), making the resync action to fail when + executed on the primary. + + Until #35332 such failures never happened because the + TransportResyncReplicationAction is skipping the reroute phase, the only + place where blocks were checked. Now with #35332 blocks are checked + during reroute and also during the execution of the transport replication + action on the primary. After some internal discussion, we decided that the TransportResyncReplicationAction should never be blocked. This action is + part of the replica to primary promotion and makes sure that replicas are in + sync and should not be blocked when the cluster state has no master or + when the index is read only. + + This commit changes the TransportResyncReplicationAction to make obvious + that it does not honor blocks. It also adds a simple test that fails if the resync + action is blocked during the primary action execution. + + Closes #35597 + +commit be8097f9ceaa41c5d43c7a9cb27b6c5993b5c24a +Author: Alan Woodward +Date: Thu Nov 22 09:20:46 2018 +0000 + + Improve docs for index_prefixes option (#35778) + + This commit moves the documentation and examples for the `index_prefixes` + option on text fields to its own file, to bring it in line with other mapping + parameters, and expands a bit on both. + +commit cfdf666672a5551cfdb497f624e1e3869760f855 +Author: David Turner +Date: Thu Nov 22 09:08:52 2018 +0000 + + [Zen2] Fix test failures in diff-based publishing (#35684) + + `testIncompatibleDiffResendsFullState` sometimes makes a 2-node cluster and + then partitions one of the nodes from the leader, which makes the leader stand + down. Then when the partition is removed the cluster re-forms but does so by + sending full cluster states, not diffs, causing the test to fail. + + Additionally `testDiffBasedPublishing` sometimes fails if a publication is + delivered out-of-order, wiping out a fresher last-received cluster state with a + less-fresh one. This is fixed here by passing the received cluster state to the + coordinator before recording it as the last-received one, relying on the + coordinator's freshness checks. + +commit 2887680acbb49fa2e0cd1fde39172ea3163c1318 +Author: Jason Tedor +Date: Wed Nov 21 19:16:28 2018 -0500 + + Avoid NPE in follower stats when no tasks metadata (#35802) + + When there is no persistent tasks metadata we could hit a null pointer + exception when executing a follower stats request. This is because we + inspect the persistent tasks metadata. Yet, if no tasks have been + registered, this is null (as opposed to empty). We need to avoid + de-referencing the persistent tasks metadata in this case. That is what + this commit does, and we add a test for this situation. + +commit b01daedfb518cc74769a013131e92c24d8ae9042 +Author: Zachary Tong +Date: Wed Nov 21 18:16:50 2018 -0500 + + [TEST] Ensure job does not trigger + + By setting the cron to 2017, we ensure it won't trigger. This makes it + easier to test because we know the job will _only_ be in STARTED, + and we can ignore INDEXING states due to transient triggers. + + Closes #35779 + +commit 78209188bcc77ad8685fbc161ea391984f8b0fee +Author: Lisa Cawley +Date: Wed Nov 21 15:01:59 2018 -0800 + + [DOCS] Adds more authorization info for CCR APIs (#35606) + +commit 927b2425b69244d0c45114f7cb4478e028f2fcbd +Author: Tal Levy +Date: Wed Nov 21 14:59:08 2018 -0800 + + [TEST] fix invalid ExplainLifecycleRequest in tests (#35804) + +commit 90a8e4b259f4d7590cf90594742cfc65e2c45e57 +Author: Benjamin Trent +Date: Wed Nov 21 16:22:04 2018 -0600 + + HLRC: ML Delete event from Calendar (#35760) + + * HLRC: Delete event from calendar + + * adjusting tests + + * adjusting code to make it more readable + +commit d95d885bae51ea352a287052593c8744a9452fe1 +Author: Marios Trivyzas +Date: Wed Nov 21 23:15:10 2018 +0100 + + SQL: Implement ISNULL(expr1, expr2) (#35793) + + Add ISNULL as an alias of IFNULL as they have the + same behaviour. Add basic test and docs. + + Closes: #35781 + +commit 3548d6a4bb8a0a31d21ae5df3d8bc1e1aab4ded0 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Nov 22 09:10:55 2018 +1100 + + [HLRC] Fix issue in equals impl for GlobalOperationPrivileges (#35721) + + This commit fixes an issue in the equals implementation for + GlobalOperationPrivileges and adds few tests. + +commit 39789d0a10d38f56b86976255e150d7b3b80df96 +Author: Igor Motov +Date: Wed Nov 21 11:04:01 2018 -1000 + + GEO: More robust handling of ignore_malformed in geoshape parsing (#35603) + + Adds an XContent sub parser class that can to wrap another + XContent parser at the beginning of an object and allow skiping + all children in case of the parsing failure. It also uses this + subparser to ignore the rest of the GeoJson shape if the + parsing fails and we need to ignore the geoshape due to the + ignore_malformed flag. + + Supersedes #34498 + + Closes #34047 + +commit 1a5553d4959d3832e3a9dff8f76d6eb1b40d4a79 +Author: Armin Braun +Date: Wed Nov 21 19:58:51 2018 +0100 + + MINOR: Cleanup Runnables in SnapshotsService (#35796) + + * Simplify complex `Runnable` by moving to `AbstractRunnable` + +commit 60db06ed1dbf35e5d6d3b22ef2ef6b13643a6570 +Author: Tal Levy +Date: Wed Nov 21 09:38:20 2018 -0800 + + [ILM] refactor ExplainLifecycleRequest to enforce indices (#35753) + + Previously, it was possible to initialize a request + that does not include any indices. This results in undefined + behavior when generating the URL path to send to ES. By + enforcing indices to be defined, users will be more explicit + about the intention. + +commit 3bee25cb700208311ef751b1f460342bdba4ec70 +Author: Nick Knize +Date: Wed Nov 21 11:15:01 2018 -0600 + + [GEO] Add support to ShapeBuilders for building Lucene geometry (#35707) + + * [GEO] Add support to ShapeBuilders for building Lucene geometry + + This commit adds support for building lucene geometry from the ShapeBuilders. + This is needed for integrating LatLonShape as the primary indexing approach + for geo_shape field types. All unit and integration tests are updated to + add randomization for testing both jts/s4j shapes and lucene shapes. + +commit d9c6986b7549e108ae45d95691fbafcd8c294dbf +Author: Gordon Brown +Date: Wed Nov 21 10:06:57 2018 -0700 + + Ensure all Steps have Javadoc (#35757) + + Adds or corrects Javadoc on subclasses of Step. + +commit c81634725383e1bd409639088297a7d404bf4468 +Author: Yannick Welsch +Date: Wed Nov 21 16:31:44 2018 +0100 + + Disable testClusterJoinDespiteOfPublishingIssues for Zen2 + + This test is failing sometimes with Zen2 due to the lack of lag detection. + Zen1 does not have this problem as it only considers a join as valid if the + corresponding cluster state update is successfully published and committed + on the joining node. + +commit e179bd393dadc5151ea28ca7ec125743a437ca51 +Author: Marios Trivyzas +Date: Wed Nov 21 17:07:07 2018 +0100 + + SQL: Implement IFNULL variant of COALESCE (#35762) + + IFNULL is a MySQL variant (also used in other DBs) which + takes only 2 arguments and returns the first one that is not null. + + Closes: #35749 + +commit a056bd8c1c457348223bda6bb806a39bdc063b2f +Author: Andrey Ershov +Date: Wed Nov 21 17:03:26 2018 +0100 + + [Zen2] Move ClusterState fields to be persisted to ClusterState.MetaData (#35625) + + Today we have a way to atomically persist global MetaData and + IndexMetaData to disk when new ClusterState is received. All other + ClusterState fields are not persisted. + However, there are other parts of ClusterState that should be + persisted, namely: + + version + term + lastCommittedConfiguration + lastAcceptedConfiguration + votingTombstones + + version is changed frequently, other fields are not. We decided + to group term, lastCommittedConfiguration, + lastAcceptedConfiguration and votingTombstones into + CoordinationMetaData class and make CoordinationMetaData a field + inside MetaData. + MetaData.toXContent and MetaData.fromXContent should take care of + CoordinationMetaData. + version stays as a top level field in ClusterState and will be + persisted as part of Manifest in a follow-up commit. + Also MetaData.isGlobalStateEquals should be extended to include + coordinationMetaData in comparison. + + This commit favors exposing getters, such as getTerm directly in + ClusterState to avoid massive code changes. + + An example of CoordinationMetaState.toXContent: + + { + "term": 1, + "last_committed_config": [ + "TiIuBcbBtpuXyDDVHXeD", + "ZIAoVbkjjLPLUuYLaTkw" + ], + "last_accepted_config": [ + "OwkXbXZNOZPJqccdFHdz", + "LouzsGYwmQzpeQMrboZe", + "fCKGRZdjLTqzXAqPUtGL", + "pLoxshjpJXwDhbgjfYJy", + "SjINLwFIlIEFZCbjrSFo", + "MDkVncJEVyZLJktopWje" + ] + } + +commit 03f003733dd98dea5192d4444e11184aeeaa2c44 +Author: Costin Leau +Date: Wed Nov 21 17:35:15 2018 +0200 + + SQL: Perform lazy evaluation of mismatched mappings (#35676) + + Move away from performing eager, fail-fast validation of mismatched + mapping to a lazy evaluation based on the fields actually used in the + query. This allows queries to run on the parts of the indices that + "work" which is not just convenient but also a necessity for large + mappings (like logging) where alignment is hard/impossible to achieve. + + Fix #35659 + +commit 26cc8ff8c3664e059071dc9ed539296b83e9e61a +Author: Alan Woodward +Date: Wed Nov 21 15:27:11 2018 +0000 + + Add pointer to the index-phrases option in shingle filter docs (#35771) + + We should be discouraging the use of shingle filters and instead pointing users to the + index-phrases parameter on text fields. + +commit 592a909ba7cba3982a5f4add32f88d7d46871daa +Author: Albert Zaharovits +Date: Wed Nov 21 17:15:14 2018 +0200 + + `manage_token` privilege for `kibana_system` + + Creates the manage_token cluster privilege and adds it to the + kibana_system role. This is required if kibana were to use the token + service for its authenticator process. + Because kibana_system already has manage_saml this effectively + only adds the privilege to create tokens. + +commit 3c466d29d9d745cf8c81b9a005315332be1eb378 +Author: Ioannis Kakavas +Date: Wed Nov 21 17:05:30 2018 +0200 + + Add missing tag for get-privileges Docs + +commit 25f83ae08c5e8ef8c98bc20067fde6ef212c3d59 +Author: Ioannis Kakavas +Date: Wed Nov 21 16:38:17 2018 +0200 + + [HLRC] Add support for get application privileges API (#35556) + + This commits adds support for the Get Application Privileges + API to the HLRC + + Relates: #29827 + +commit 6ac0cb1842b85d5a93f036c9d0790ecb4b2b1fc3 +Merge: 8939a7894fb bdf632b6f99 +Author: Andrey Ershov +Date: Wed Nov 21 15:12:58 2018 +0100 + + Merge branch master into zen2 + + 2 types of conflicts during the merge: + 1) Line length fix + 2) Classes no longer extend AbstractComponent + +commit f0a3d326022b92ead0a2b2bf9c5155a77d8ad8c8 +Author: Costin Leau +Date: Wed Nov 21 15:45:59 2018 +0200 + + SQL: Introduce INTERVAL support (#35521) + + Introduce INTERVAL as a DataType + Add INTERVAL to the grammar which supports the standard SQL declaration + (without precision): + > INTERVAL '1 23:45:01.123456789' DAY TO SECOND + but also number for single unit intervals: + > INTERVAL 1 YEAR + as well as the plurals of the units: + > INTERVAL 2 YEARS + Interval are internally supported as just another Literal being backed + by java.time.Period and java.time.Duration + Move JDBC away from JDBCType enum to SQLType interface + Refactor DataType by moving it into server core and adding dedicated (and + much simpler) JDBC driver type + Improve internal JDBC conversion by normalizing on the DataType + Rename JDBC columnInfo to JdbcColumnInfo to differentiate between it and + the SQL ColumnInfo + + Fix #29990 + +commit 8939a7894fb507345f9a6b2847cd62b37bad8f9f +Author: Yannick Welsch +Date: Wed Nov 21 14:43:33 2018 +0100 + + Zen2: Move disruption tests to Zen2 (#35724) + + - Moves disruption tests to Zen2 + - Registers a few missing settings + - Removes .put(TestZenDiscovery.USE_ZEN2.getKey(), true) from tests where Zen2 is now enabled + by default through the parent test class + - Moves QuorumGatewayIT back to Zen1, as it is not stable with Zen2 as it currently relies on + dangling indices due to the lack of proper CS persistence, which triggers secondary failures + +commit bdf632b6f99d851ad5963d765375751757265a5d +Author: Armin Braun +Date: Wed Nov 21 13:50:17 2018 +0100 + + SNAPSHOTING+MINOR: Simplify SnapshortShardService (#35769) + +commit 9b2ab064cface6e412de17e2bbcb423e5595ebaa +Author: Martijn van Groningen +Date: Wed Nov 21 10:52:03 2018 +0100 + + [HLRC] Added support for CCR Resume Follow API (#35638) + + This change also adds documentation for the Resume Follow API + + Relates to #33824 + +commit 7a210342abc258ca47b32f07176d839fe7074038 +Author: Armin Braun +Date: Wed Nov 21 10:33:50 2018 +0100 + + TESTS: Remove Dead Code in Disruption Tests (#35768) + + * Neither this class nor the constructor are used anywhere + +commit 1cb578b43570a953e1cdd102015b6e787bf65b56 +Author: Dimitrios Liappis +Date: Wed Nov 21 11:32:29 2018 +0200 + + [DOCS] Update sysctl instructions for Docker on Mac (#35755) + + Recent Docker for Mac releases[1] have a different path to the tty for + accessing the console of the xhyve vm, required for altering the + `vm.max_map_count` sysctl. + + Update instructions on how to enter the xhyve vm for altering the + `vm.max_map_count` sysctl setting on Docker for Mac. + + Closes #34817 + + [1] + https://forums.docker.com/t/is-it-possible-to-ssh-to-the-xhyve-machine/17426/13 + +commit 6638708b5624e49cdf396127140c76032a883abf +Author: Christoph Büscher +Date: Wed Nov 21 10:29:08 2018 +0100 + + Remove deprecated QueryStringQueryBuilder#splitOnWhiteSpace (#35763) + + This parameter has been deprecated and was ignored since 6.0, so its Java API + methods can be removed. + +commit 5847f8379c6d960517a6a70fb8f1dec87dce2221 +Author: Christoph Büscher +Date: Wed Nov 21 10:28:31 2018 +0100 + + Move ScoreAccessor to test-framework (#35766) + + This class is only used by RandomScoreFunctionIT and the MockScriptEngine, so it + shouldn't be part of the server codebase. + +commit 65d8fdf3da084911890c9d03e1d3556c913c5028 +Author: Luca Cavanna +Date: Wed Nov 21 10:21:45 2018 +0100 + + Clean up PutLicenseResponse (#35689) + + This commit removes the parsing code from the PutLicenseResponse server variant, and the toXContent portion from the corresponding client variant. + + Relates to #35547 + +commit 778550a97db4a84d31e475155a7cec51e543842b +Author: Luca Cavanna +Date: Wed Nov 21 10:21:10 2018 +0100 + + Clean up StartBasicResponse (#35688) + + This commit removes the parsing code from the PostStartBasicResponse server variant. It also makes the server response implement StatusToXContent which allows us to save a couple of lines of code in the corredponding REST action. + + Relates to #35547 + +commit 4f857c4f8d43b77a6b2741122fb572fed84b5844 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Wed Nov 21 09:10:37 2018 +0000 + + [HLRC][ML] Add ML revert model snapshot API (#35750) + + Relates to #29827 + +commit f6a43b59392b314ae277369c2e7fe50ea90c9115 +Author: Alan Woodward +Date: Wed Nov 21 09:00:48 2018 +0000 + + Add a prebuilt ICU Analyzer (#34958) + + The ICU plugin provides the building blocks of an analysis chain, but doesn't actually have a prebuilt analyzer. It would be a better for users if there was a simple analyzer that they could use out of the box, and also something we can point to from the CJK Analyzer docs as a superior alternative. + + Relates to #34285 + +commit e8ec4fad7b5f1be7dc378e1c8591193ac2d9fe21 +Author: Ioannis Kakavas +Date: Wed Nov 21 09:32:56 2018 +0200 + + [DOCS] Adjust Invalidate Token REST API docs (#35622) + + - Renames API to Invalidate Token + - Explicitly calls out the possibility to invalidate refresh tokens + via this API + +commit a6647a20a9a5ff2b28ee878720154d6d3823d8db +Author: Martijn van Groningen +Date: Wed Nov 21 07:48:29 2018 +0100 + + [HLRC] Added support for CCR Unfollow API (#35693) + + This change also adds documentation for the Unfollow API + + Relates to #33824 + +commit 30c54225611e3c2fc753305ab82ed45061778388 +Author: Tim Vernum +Date: Wed Nov 21 14:33:10 2018 +1100 + + Move XContent generation to HasPrivilegesResponse (#35616) + + The RestHasPrivilegesAction previously handled its own XContent + generation. This change moves that into HasPrivilegesResponse and + makes the response implement ToXContent. + + This allows HasPrivilegesResponseTests to be used to test + compatibility between HLRC and X-Pack internals. + + A serialization bug (cluster privs) was also fixed here. + +commit ff03443ab945ad876fedaabf2b447146b9d6a2f2 +Author: Christoph Büscher +Date: Wed Nov 21 03:49:49 2018 +0100 + + Fix problem with MatchNoDocsQuery in disjunction queries (#35726) + + Queries across multiple fields generate MatchNoDocsQuerys for fields that are + unmapped. In certain situation this can lead to erroneous behaviour, + for example when an umapped field is used in a query_string query across + several fields. If some of the tokens in the query string get eliminated by an + analyzer on the mapped fields, the same token will currently generate + MatchNoDocsQuerys combined into a disjunction, which in turn + leads to no matches in the overall query. Instead we should simply not add + MatchNoDocsQuerys to those disjunctions. + + Closes #34708 + +commit fa6d8a5004abb745b63ee8c77d6d09e26e019d85 +Author: lcawl +Date: Tue Nov 20 17:31:01 2018 -0800 + + [DOCS] Fixes navtitle for CCR API + +commit b1818dbdcee2847b821ab75d7f72c24f5f0a0046 +Author: Marios Trivyzas +Date: Wed Nov 21 01:43:05 2018 +0100 + + SQL: [docs] Add documentation for COALESCE (#35740) + + Follows: #35253 + +commit 04a087fe2f19e83874bb311688961b64ddf062e5 +Author: Lisa Cawley +Date: Tue Nov 20 16:23:01 2018 -0800 + + [DOCS] Adds authorization info for CCR APIs (#35557) + +commit 84db1e42c00e02bd161989fb0716911e88f206f5 +Author: Benjamin Trent +Date: Tue Nov 20 16:40:31 2018 -0600 + + HLRC: ML Get Calendar Events (#35747) + + * HLRC: ML Get Calendar Events + + * Addressing PR comments + +commit 643bb20137182582478584f63ae9d31101b7d9b8 +Author: Mayya Sharipova +Date: Tue Nov 20 16:10:06 2018 -0500 + + Add a new query type - ScriptScoreQuery (#34533) + + * Add a new query type - ScriptScoreQuery + + script_score query uses script to calculate document scores. + Added as a substitute for function_score with an intentation + to deprecate function_scoreq query. + + ```http + GET /_search + { + "query": { + "script_score" : { + "query": { + "match": { "message": "elasticsearch" } + }, + "script" : { + "source": "Math.log(2 + doc['likes'].value)" + }, + "min_score" : 2 + } + } + } + ``` + + Add several functions to painless to be used inside script_score: + + double rational(double, double) + double sigmoid(double, double, double) + double randomNotReproducible() + double randomReproducible(String, int) + + double decayGeoLinear(String, String, String, double, GeoPoint) + double decayGeoExp(String, String, String, double, GeoPoint) + double decayGeoGauss(String, String, String, double, GeoPoint) + + double decayNumericLinear(String, String, String, double, double) + double decayNumericExp(String, String, String, double, double) + double decayNumericGauss(String, String, String, double, double) + + double decayDateLinear(String, String, String, double, JodaCompatibleZonedDateTime) + double decayDateExp(String, String, String, double, JodaCompatibleZonedDateTime) + double decayDateGauss(String, String, String, double, JodaCompatibleZonedDateTime) + + Date functions only works on dates in the default format and default time zone + +commit c07ad677181b57895b96529b1e7136bd60798ca3 +Author: Ryan Ernst +Date: Tue Nov 20 08:58:47 2018 -0800 + + Add 6.5.2 version + +commit 4bda469861a7ebc46d8144f3380d9b7070336638 +Author: Gordon Brown +Date: Tue Nov 20 12:29:48 2018 -0700 + + Fix line lengths in misc other files (#35650) + +commit 953c8586df558de1aa4ac90cfcf1f8fdb946598b +Author: Vladimir Dolzhenko +Date: Tue Nov 20 20:05:23 2018 +0100 + + Fix PrimaryAllocationIT#testForceStaleReplicaToBePromotedToPrimary (#35728) + + + Closes 35497 + +commit 328448e8cdf43ddaa6d3931ba6a2056797f6d817 +Author: Alpar Torok +Date: Tue Nov 20 20:35:53 2018 +0200 + + Implement a property to allow for splitting tests. (#35473) + +commit 17780ce07e4c709ce2e9ced6f00e6919e3c75334 +Author: Gordon Brown +Date: Tue Nov 20 11:32:41 2018 -0700 + + Add HLRC docs for Delete Lifecycle Policy (#35664) + + Adds documenatation for the Delete Lifecycle Policy API to the HLRC + documentation. + +commit 89cf4a7397b891e48369e5873921be4635c1b1de +Author: Armin Braun +Date: Tue Nov 20 17:50:08 2018 +0100 + + NETWORKING: Fix IpFiltering Test (#35730) + + * The port assigned to all loopback interfaces doesn't necessarily have to be the same for ipv4 and ipv6 + => use actual address from profile instead of just port + loopback in test + * Closes #35584 + +commit e91f404d1650573b7af70b7df8f5bc1f831f0f5a +Author: Christoph Büscher +Date: Tue Nov 20 16:07:17 2018 +0100 + + Remove remains of 'auto_generate_phrase_queries' (#35735) + + This parameter in the `query_string` query was deprecated in 6.0 and ignored + since then. Its API methods and remaining uses can be removed in the upcoming + major version. + + Relates to #35734 + +commit d061b3999aba6c8921dcb3a617f80b174647560e +Author: Tal Levy +Date: Tue Nov 20 07:05:27 2018 -0800 + + [ILM] HLRC-ILM Retry Lifecycle Policy docs (#35715) + + this adds documentation for the retry method in the + high-level-ilm-rest-client. + + this PR also renames retryLifecycleStep to retryLifecyclePolicy in the index-lifecycle-client + +commit 33c713ba600f673767adf82aa69e8dfeae4720ff +Author: Armin Braun +Date: Tue Nov 20 15:36:01 2018 +0100 + + TESTS: More Logging in LongGcDisruptionTests (#35702) + + * The existing logging is not helpful enough to track down which threads hang, we need the hanging thread's stacktraces too + * Relates #35686 + +commit 7657e6d274bc4ac48f439db82b1a1336651af7de +Author: Benjamin Trent +Date: Tue Nov 20 08:15:21 2018 -0600 + + HLRC ML Add Event To Calendar API (#35704) + + * HLRC: ML Adding Post event to calendar api + + * Fixing tests and serialization + + * removing unused import + +commit d707838c026d3ffe0113d780da9badd40d33a2b3 +Author: Benjamin Trent +Date: Tue Nov 20 07:43:34 2018 -0600 + + HLRC: ML Delete job from calendar (#35713) + +commit 8892408edd8088cdde8f21ecbf21d1cfcf72efb3 +Author: Armin Braun +Date: Tue Nov 20 14:09:15 2018 +0100 + + SNAPSHOT: Fix Leaking Snapshot Task in IT (#35657) + + * SNAPSHOT: Fix Leaking Snapshot Task in IT + + * If the test randomization causes the request to not wait for snapshot completion + then the deleting of the still in progress snapshot could fail in the after hook + * Fixed by deleting the snapshot in the test + * Closes #35642 + +commit 844483a99a4e0a32705aa61d91e9a37bf0ddaec0 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Tue Nov 20 12:18:29 2018 +0000 + + [HLRC][ML] Add ML update model snapshot API (#35537) (#35694) + + + Relates to #29827 + +commit 8659af68e0e6cdea45bee7755b49fc0d2e248b73 +Author: Alpar Torok +Date: Tue Nov 20 13:02:33 2018 +0200 + + Auto skip license headers on no source (#35640) + + * Unmute BuildExamplePluginsIT + + * Skip licenseHeaders when there are no sources + +commit 7b2d547e8ea01cf2346285d5e7293a22041b6a0f +Author: Takuro Wada +Date: Tue Nov 20 18:08:32 2018 +0900 + + [Docs] Delete inappropriate backtick (#35722) + +commit a5f5ceb869ff62877ad0725dbaa174167bc4e1f9 +Author: Jim Ferenczi +Date: Tue Nov 20 08:09:14 2018 +0100 + + Remove remaining line length violations for o.e.index (#35652) + + This commit removes line length violations in the classes under + org.elasticsearch.index. + +commit 29ef44284177e9909bd8d49eec1347652823ce48 +Author: Simon Willnauer +Date: Tue Nov 20 08:03:24 2018 +0100 + + Add a `_freeze` / `_unfreeze` API (#35592) + + This commit adds a rest endpoint for freezing and unfreezing an index. + Among other cleanups mainly fixing an issue accessing package private APIs + from a plugin that got caught by integration tests this change also adds + documentation for frozen indices. + Note: frozen indices are marked as `beta` and available as a basic feature. + + Relates to #34352 + +commit a989b675b521281a6d2837dac8f619aaf6587e8a +Author: Tim Brooks +Date: Mon Nov 19 20:38:49 2018 -0700 + + Remove NPE from IndexFollowingIT (#35717) + + Currently there is a common NPE in the IndexFollowingIT that does not + indicate the test failing. This is when a cluster state listener is + called and certain index metadata is not yet available. + + This commit checks that the metadata is not null before performing the + logic that depends on the metadata. + +commit d367894c313a6c97f4b2ebf46fe70adda6023352 +Author: Tim Vernum +Date: Tue Nov 20 13:03:22 2018 +1100 + + DOCS: Add password_hash & refresh to Put User API (#35574) + + PR #35242 formalised support for the password_hash field in the body + of the Put User security API. + Since this field is now validated and tested, it can also be + documented. + + The Put User API also supports a "refresh" query parameter that was + not documented. This commit adds it to the docs. + +commit 3bba9a56de54ce80bac34806eddcdb035aaa3239 +Author: Tim Vernum +Date: Tue Nov 20 12:51:01 2018 +1100 + + TEST: Enable skipped test (#35682) + + This test was skipped due to infr complications during the work + to open x-pack, but wasn't enabled again when the issue was resolved. + + Relates: #29890 + +commit 220c175484c3d0627d6f2c1daa837dd2fb386469 +Author: Jason Tedor +Date: Mon Nov 19 20:40:58 2018 -0500 + + Provide links to auto-follow docs (#35674) + + This commit adds a link to the auto-follow docs from the CCR getting + started docs and the relevant CCR APIs. + +commit e012ac46491526b75ad08ce2b556d0b3a899d9ff +Author: Gordon Brown +Date: Mon Nov 19 17:17:01 2018 -0700 + + Add HLRC docs for ILM Start/Stop/Status (#35672) + + Adds HLRC documentation for Start ILM, Stop ILM, and ILM Status APIs, as + these APIs are tightly linked to each other. + +commit bf02897173f46a784947617fd44d2739b8da6713 +Author: Gordon Brown +Date: Mon Nov 19 16:30:32 2018 -0700 + + Fix doc link for HLRC async Put Lifecycle API (#35656) + + The link to the HLRC documentation for the async Put Lifecycle + method was never updated to the correct link. This commit fixes that + link. + +commit 90c0d444cf3ec8a265a1b56d7ade3970e0b44625 +Author: Tal Levy +Date: Mon Nov 19 13:26:20 2018 -0800 + + [ILM] DOCS: clarify interaction between rollover and min_age (#35663) + + This adds an example to drive home the semantics of `min_age` and explain + how actions from one phase must complete before `min_age` is even tested + + closes #34020. + +commit 7a779a998801e2a055b53f6c11917e911e570415 +Author: Benjamin Trent +Date: Mon Nov 19 14:23:49 2018 -0600 + + [TEST] fixing ClusterApplierService#testClusterStateUpdateLogging() (#35714) + +commit 47ada69c465bc2598c95887c1768d2c124c2430b +Author: Yannick Welsch +Date: Mon Nov 19 21:15:29 2018 +0100 + + Zen2: Move most integration tests to Zen2 (#35678) + + Zen2 is now feature-complete enough to run most ESIntegTestCase tests. The changes in this PR + are as follows: + - ClusterSettingsIT is adapted to not be Zen1 specific anymore (it was using Zen1 settings). + - Some of the integration tests require persistent storage of the cluster state, which is not fully + implemented yet (see #33958). These tests keep running with Zen1 for now but will be switched + over as soon as that is fully implemented. + - Some very few integration tests are not running yet with Zen2 for other reasons, depending on + some of the other open points in #32006. + +commit 3a232d789751bbd64106e037fa120522407374b9 +Author: lcawl +Date: Mon Nov 19 11:42:45 2018 -0800 + + [DOCS] Adds anchor for SAML Kibana basic info + +commit f9ecd0c49e7b9dc9594f798269f8376596a57589 +Author: Andrey Ershov +Date: Mon Nov 19 19:49:44 2018 +0100 + + [Zen2] Write manifest file (#35049) + + Elasticsearch node is responsible for storing cluster metadata. + There are 2 types of metadata: global metadata and index metadata. + `GatewayMetaState` implements `ClusterStateApplier` and receives all + `ClusterStateChanged` events and is responsible for storing modified + metadata to disk. + + When new `ClusterStateChanged` event is received, `GatewayMetaState` + checks if global metadata has changed and if it's the case writes new + global metadata to disk. After that `GatewayMetaState` checks if index + metadata has changed or there are new indices assigned to this node and + if it's the case writes new index metadata to disk. Atomicity of global + metadata and index metadata writes is ensured by `MetaDataStateFormat` + class. + + Unfortunately, there is no atomicity when more than one metadata changes + (global and index, or metadata for two indices). And atomicity is + important for Zen2 correctness. + This commit adds atomicity by adding a notion of manifest file, + represented by `MetaState` class. `MetaState` contains pointers to + current metadata. + More precisely, it stores global state generation as long and map from + `Index` to index metadata generation as long. Atomicity of writes for + manifest file is ensured by `MetaStateFormat` class. + + The algorithm of writing changes to the disk would be the following: + + 1. Write global metadata state file to disk and remember + it's generation. + 2. For each new/changed index write state file to disk and remember + it's generation. For each not-changed index use generation from + previous manifest file. If index is removed or this node is no longer + responsible for this index - forget about the index. + 3. Create `MetaState` object using previously remembered generations and + write it to disk. + 4. Remove old state files for global metadata, indices metadata and + manifest. + + Additonally new implementation relies on enhanced `MetaDataStateFormat` + failure semantics, `applyClusterState` throws IOException, whose + descendant `WriteStateException` could be (and should be in Zen2) + explicitly handled. + +commit 80bcf7c7048b8dca8e815e42358556f93cf239b6 +Author: Benjamin Trent +Date: Mon Nov 19 12:49:07 2018 -0600 + + [ML]: adjusting post backport of (#35645) (#35696) + +commit a0e697285b5530d4f9fc9b6578e365493fd424ae +Author: Mayya Sharipova +Date: Mon Nov 19 12:58:58 2018 -0500 + + Correct errors - HRLC TermVectorRequest (#35705) + + Relates to #35266, #35614 + +commit 214bc967389a8b2910229376578a23757ccf12b7 +Author: Benjamin Trent +Date: Mon Nov 19 11:41:49 2018 -0600 + + HLRC: ML Add Job to Calendar API (#35666) + +commit aaeb47d1f7ce4c106a7f24b070f24117f0e0717f +Author: Mayya Sharipova +Date: Mon Nov 19 12:24:31 2018 -0500 + + HLRC for _mtermvectors (#35266) + + relates to #27205 + +commit b2057138a72dd221655fc868fee555c018c8c838 +Author: Gordon Brown +Date: Mon Nov 19 09:51:32 2018 -0700 + + Remove AbstractComponent from AbstractLifecycleComponent (#35560) + + AbstractLifecycleComponent now no longer extends AbstractComponent. In + order to accomplish this, many, many classes now instantiate their own + logger. + +commit 156b3cae15139c4e81a92b62bab2c3ec9777b345 +Author: Itamar Syn-Hershko +Date: Mon Nov 19 17:27:43 2018 +0200 + + [Docs] Fix filter context in script-query.asciidoc (#35677) + + The docs say script queries are typically used in a filter context but + the example uses a boolean must clause. + +commit a8433a3164c8c51fd598fa0d5743322d6cf6d705 +Author: Tanguy Leroux +Date: Mon Nov 19 16:30:20 2018 +0100 + + [Docs] Fix `service_api_key` in PagerDuty documentation (#35589) + + closes #35572 + +commit 036c15d8a38857ec82b248c0d5dbb6972dbc5b89 +Author: Nik Everett +Date: Mon Nov 19 10:17:05 2018 -0500 + + Build: Fix official plugins list (#35661) + + The list of official plugins accidentally included `qa` projects like, + well, `qa` and `amazon-ec2`. This changes the mechanism that we use to + build the list and adds a test to catch this. + + Closes #35623 + +commit 6d4a3f8fce5d4445c7dff2b340433063e1b01c1c +Author: Benjamin Trent +Date: Mon Nov 19 08:29:53 2018 -0600 + + Removes two unused AnalysisConfig options (#35645) + + * ML: Removing result_finalization_window && overlapping_buckets + + * Reverting bad method deletions + + * Setting to current before backport to try and get a green build + + * fixing testBuildAutodetectCommand test + + * disabling bwc tests for backport + +commit 450db7fcf7b99066b42507be15727856765729ff +Author: Ekal Golas +Date: Mon Nov 19 06:21:01 2018 -0800 + + [Tests] Fix slowness of AutoDateHistogramAggregatorTests (#35072) + + Randomize test assertion and test set size instead of asserting on an + exhaustive list of dates with fixed test set size. Also refactor common + objects used to avoid recreating them, avoid date to string conversion + and reduce duplicate test code + + Closes #33181 + +commit bc7dea4480bba19f7c4f864289d7d0fad3d8eac3 +Author: Benjamin Trent +Date: Mon Nov 19 08:03:34 2018 -0600 + + ML: changing automatic check_window calculation (#35643) + + * ML: changing automatic check_window calculation + + * adding docs on how we calculate the default + +commit f70b7dc158875ec58e137a0878b589bfc53f3fa6 +Author: David Roberts +Date: Mon Nov 19 12:55:33 2018 +0000 + + Mute DataTypeConversionTests.testConversionToDate + + Due to https://github.com/elastic/elasticsearch/issues/35683 + +commit 6d8af9731d96cc092a5cbb72b0cec366a451997b +Author: Peter Dyson +Date: Mon Nov 19 22:21:59 2018 +1000 + + [Docs] Warn about searching across all fields wt. `query_string` (#35570) + + Warn about potential performance impact when a large number of fields + is used with query string query and no default field. + +commit fda59ff96983394622235926465d9058fbfd3076 +Author: Alpar Torok +Date: Mon Nov 19 14:01:32 2018 +0200 + + The default of tests.jvms should not be capped (#35259) + + Default to the number of physical cores across all sockets instead. + +commit ae6a33237fca08094f6e38004ef5c9dc2339bc55 +Author: Ignacio Vera +Date: Mon Nov 19 11:25:35 2018 +0100 + + HLRC: Add ML delete filter action (#35382) + + * HLRC: Add ML delete filter action + + It adds delete ML filter action to the high level rest client. + + Relates #29827 + +commit 2ee25c858d622a5caa760e79ff3f5010e43fab68 +Author: David Roberts +Date: Mon Nov 19 09:35:29 2018 +0000 + + Mute CronEvalToolTests.testEnsureDateIsShownInRootLocale + + Due to https://github.com/elastic/elasticsearch/issues/35687 + +commit c468d928b8a212bbfc2037c17fd8d3e9ba2df308 +Author: Martijn van Groningen +Date: Mon Nov 19 07:36:24 2018 +0100 + + Remove line length violations in o.e.indices package (#35647) + + Relates to #34884 + +commit 4119409b6d8db045bb231756986ca225c3f76284 +Author: Costin Leau +Date: Sun Nov 18 23:35:18 2018 +0200 + + SQL: Improve validation of unsupported fields (#35675) + + Fix bug in Analyzer that caused it to report unsupported fields only + when declared in projections. The rule has been extended to all field + declarations. + + Fix #35673 + +commit f8e333b117915add0e68aeaac0640b50a85ff5fb +Author: Costin Leau +Date: Sat Nov 17 15:30:27 2018 +0200 + + SQL: Move internals from Joda to java.time (#35649) + + Remove/Limit usage of Joda through-out the processors and functions + Use ZonedDateTime wherever possible instead of long/tzId + + Fix #35633 + +commit bb51cdb6de0d94d18bea9999e80fe93c481dd8e4 +Author: Gordon Brown +Date: Fri Nov 16 18:16:30 2018 -0700 + + Add HLRC docs for Get Lifecycle Policy (#35612) + + Adds docs for the Get Lifecycle Policy API to the HLRC documentation. + +commit cce9648f9d8dc8a81c771b6565a188a5db924533 +Author: Gordon Brown +Date: Fri Nov 16 17:42:48 2018 -0700 + + Align RolloverStep's name with other step names (#35655) + + RolloverStep previously had a name of "attempt_rollover", which was + inconsistent with all other step names due it its use of an underscore + instead of a dash. + +commit 3c03975ce396ff3e6f27b79d26e77a4cf1081f67 +Author: Ryan Ernst +Date: Fri Nov 16 15:46:27 2018 -0800 + + Watcher: Use joda method to get local TZ (#35608) + + This commit switches from using java util's default timezone method to + using joda. The former can cause problems when the string representation + of the timezone is unknown to joda. + + closes #35518 + +commit 22933ebcd5cab852ad144da607253a13cdd98017 +Author: Lee Hinman +Date: Fri Nov 16 14:11:39 2018 -0700 + + Fix line length for org.elasticsearch.action.* files (#35607) + + This removes the checkstyle suppressions for things in the `action` package. + + Relates to #34884 + +commit 022726011ce1d9e4748fd39293a3aeae6954b3ac +Author: Arthur Gavlyukovskiy +Date: Fri Nov 16 13:10:32 2018 -0800 + + Remove use of AbstractComponent in server (#35444) + + Removed extending of AbstractComponent and changed logger usage to + explicit declaration. Abstract classes still have logger + declaration using this.getClass() in order to show implementation class + name in its logs. + + See #34488 + +commit c6a0904e0e68524d2b9d145e85a39cb50daef20a +Author: Julie Tibshirani +Date: Fri Nov 16 13:04:43 2018 -0800 + + Deprecate types in count and msearch. (#35421) + + * Deprecate types in count requests. + * Move RestCountAction to the 'search' package. + * Deprecate types in multi search requests. + * Add tests for types deprecation in the _search endpoint. + +commit 4fea6b6f9b6d49bee3ec37d9cedc58045c3b6288 +Author: Julie Tibshirani +Date: Fri Nov 16 13:03:32 2018 -0800 + + Refactor an ambigious TermVectorsRequest constructor. (#35614) + +commit d62bbca56da771afb72ffcf1142c72ecbf34ea94 +Author: Jack Conradson +Date: Fri Nov 16 12:33:01 2018 -0800 + + [Scripting] Use Number as a return value for BucketAggregationScript (#35653) + + This change fixes #35351. Users were no longer able to return types of numbers other than doubles for bucket aggregation scripts. This change reverts to the previous behavior of being able to return any type of number and having it converted to a double outside of the script. + +commit baf33b31626f13521d4862d0b700c7c6d728bb6f +Author: Jernej Klancic +Date: Fri Nov 16 12:50:18 2018 -0700 + + Removes AbstractComponent from several classes (#35566) + + Removes inhertiting from AbstractComponent for some classes (mostly + in the plugins module). + + Relates to #34488 + +commit fc63ea5357261a7b675e0ebbb6c3386d1269601d +Author: debadair +Date: Fri Nov 16 10:49:55 2018 -0800 + + [DOCS] Add beta warning to ILM pages. (#35571) + +commit 40ba4de5e64cb562a1a4c0f36efc5f797d9dc6ef +Author: Julie Tibshirani +Date: Fri Nov 16 08:59:04 2018 -0800 + + Deprecate types in validate query requests. (#35575) + +commit 6a8fe49f0caaa36b1768d16bce57d98b89edd027 +Author: Alpar Torok +Date: Fri Nov 16 11:05:43 2018 +0200 + + Unmute BuildExamplePluginsIT + +commit b7dcd306ff6293ff4f0e466aa618a2e2e25244b7 +Author: Tanguy Leroux +Date: Fri Nov 16 15:39:23 2018 +0100 + + Revert "AwaitsFix the RecoveryIT suite - see #35597" + + This reverts commit f9aff7b + +commit d3d7c0158e2e79db53c3183fe3f81861e73a1b37 +Author: Tanguy Leroux +Date: Fri Nov 16 15:38:40 2018 +0100 + + Revert "[RCI] Check blocks while having index shard permit in TransportReplicationAction (#35332)" + + This reverts commit 31567cefb4dc46c146c65c58204af825b491b743. + +commit 76b77dbaa4c9a67a87ea8ff7626c5504d8f63782 +Author: Tom Callahan +Date: Fri Nov 16 09:37:42 2018 -0500 + + Remove remaining line length violations for o.e.action.admin.cluster (#35156) + + This inserts newlines in order to reduce line lengths in the + o.e.action.admin.cluster package to 140 characters or less. This + also remves the checkstyle suppressions for affected files. + + Relates #34884, #34923 + +commit e360c600d0829977e24ff97a06687f763a226b73 +Author: Benjamin Trent +Date: Fri Nov 16 06:50:12 2018 -0600 + + ML: Adjusing BWC version post backport to 6.6 (#35605) + +commit 797cffbb682d7583bf3a05440a94d43b42afaa25 +Author: Martijn van Groningen +Date: Fri Nov 16 10:17:12 2018 +0100 + + [TEST] Replace fields in response with actual values + + Relates to #35496 + +commit 2da4bc85cf8bc8cc8f01e2d716fc2b6e25392d40 +Author: Christoph Büscher +Date: Fri Nov 16 09:35:22 2018 +0100 + + Remove usages of CharSequence in Sets (#35501) + + The javadocs of the CharSequence interface state that not all of its + implementations define the general contracts of the Object#equals and + Object#hashCode methods, therefore it is dangerous to use different CharSequence + instances as elements in a set or as keys in a map. While we probably mostly use + Strings in sets, in some places this is not enforced. To prevent this from + accidentally happening, this change replaces all occurances of Set + which are currently mostly used in the completion suggester code with the more + concrete usage of Set. + +commit f9aff7b181d330d7b801cf32ffe15e57920d95ce +Author: David Turner +Date: Thu Nov 15 17:38:31 2018 +0000 + + AwaitsFix the RecoveryIT suite - see #35597 + +commit 903d27a8fd8cc8b490a6d10810d4bf417d6b3b04 +Author: Jim Ferenczi +Date: Fri Nov 16 09:08:07 2018 +0100 + + Fix phrase_slop in query_string query (#35533) + + This change ensures that custom `phrase_slop`s are correctly handled + in the `query_string` query. + + Fixes #35125 + +commit c6e2634bf838929947b20ad97f0c768d9d0bca61 +Author: Przemyslaw Gomulka +Date: Fri Nov 16 08:58:13 2018 +0100 + + Docs: HLRC: refactor bulk, migrate and reindex apis (#35413) + + fix typos and refactor to DRY up documentation for bulk, reindex and migration apis + relates #35345 + +commit 253c2fcc6a22d2480af922e9e2586f178896c6cf +Author: Martijn van Groningen +Date: Fri Nov 16 08:00:53 2018 +0100 + + [HLRC] AcknowledgedResponse should not extend from ToXContentObject (#35372) + + AcknowledgedResponse only needs to know how to parse xcontent to + an AcknowledgedResponse instance. There is no need to serialize + an AcknowledgedResponse instance to xcontent on the HLRC side. + +commit 87a8b99724556de7dc5e932c242e7cec0a14a3f9 +Author: Tim Vernum +Date: Fri Nov 16 13:52:06 2018 +1100 + + HLRC: Add "_has_privileges" API to Security Client (#35479) + + This adds the "hasPrivileges()" method to SecurityClient, including + request, response & async variant of the method. + + Also includes API documentation. + +commit 0cdfc4cd0aa435ee5db7777709bc83a0ff94c92e +Author: Jun Ohtani +Date: Fri Nov 16 11:45:32 2018 +0900 + + [Doc] Add clarification to boolean query (#32575) + + It isn't very clear how boosting query works. + Add explanation of positive/negative query. + +commit 3883e9bf4ca5192467c182b910034636f34225a4 +Author: Gordon Brown +Date: Thu Nov 15 17:11:31 2018 -0700 + + Split RolloverStep into Wait and Action steps (#35524) + + RolloverAction will now periodically check the rollover conditions using + the Rollover API with the dry_run option as an AsyncWaitStep, then run + the rollover itself by calling the Rollover API with no conditions, + which will always roll over, as an AsyncActionStep. This will resolve + race condition issues in policies using RolloverAction. + +commit ce35d049e9eb935027d172e2e8eb5d6c78897c22 +Author: Lee Hinman +Date: Thu Nov 15 14:09:52 2018 -0700 + + [TEST] Fix ClusterApplierServiceTests.testClusterStateUpdateLogging + + This changes the test to not use a `CountDownlatch`, instead adding an assertion + for the final logging message and waiting until the `MockAppender` has seen it + before proceeding. + + Resolves #23739 + +commit a8833b097beecc9b7fc065b9755dce2d1b28bfad +Author: Mayya Sharipova +Date: Thu Nov 15 15:10:49 2018 -0500 + + Painless Context Doc: Add min should match example (#35423) + +commit 86ef0415398ae9b58b000900e72c71cef8ad849d +Author: David Turner +Date: Thu Nov 15 20:09:22 2018 +0000 + + [Zen2] Introduce ClusterBootstrapService (#35488) + + Today, the bootstrapping of a Zen2 cluster is driven externally, requiring + something else to wait for discovery to converge and then to inject the initial + configuration. This is hard to use in some situations, such as REST tests. + + This change introduces the `ClusterBootstrapService` which brings the bootstrap + retry logic within each node and allows it to be controlled via an (unsafe) + node setting. + +commit 501c03e5292fdc9c36db880afb5baf910a9e36ee +Author: Mayya Sharipova +Date: Thu Nov 15 14:33:34 2018 -0500 + + Painless Context Doc: Add filter context example (#35305) + +commit f7ada9b29b6074d17f8f969171a3fc733877690f +Author: Benjamin Trent +Date: Thu Nov 15 13:32:45 2018 -0600 + + Add delayed datacheck to the datafeed job runner (#35387) + + * ML: Adding missing datacheck to datafeedjob + + * Adding client side and docs + + * Making adjustments to validations + + * Making values default to on, having more sensible limits + + * Intermittent commit, still need to figure out interval + + * Adjusting delayed data check interval + + * updating docs + + * Making parameter Boolean, so it is nullable + + * bumping bwc to 7 before backport + + * changing to version current + + * moving delayed data check config its own object + + * Separation of duties for delayed data detection + + * fixing checkstyles + + * fixing checkstyles + + * Adjusting default behavior so that null windows are allowed + + * Mentioning the default value + + * Fixing comments, syncing up validations + +commit c7a2c6d549935016f20645cffb1dabb9ab02ee07 +Author: Jim Ferenczi +Date: Thu Nov 15 17:52:06 2018 +0100 + + Handle IndexOrDocValuesQuery in composite aggregation (#35392) + + The `composite` aggregation can optimize its execution when the query + is a `match_all` or a `range` over the field that is used in the first source + of the aggregation. However we only check for instances of `PointRangeQuery` whereas + the range query builder creates an `IndexOrDocValuesQuery`. This means that + today the optimization does not apply to `range` query even if the code could handle it. + This change fixes this issue by extracting the index query inside `IndexOrDocValuesQuery`. + +commit 90b38d9b3d383017cc7a34853d3f28018899f4a9 +Author: Lee Hinman +Date: Thu Nov 15 09:29:26 2018 -0700 + + [ILM] Avoid NullPointerException in CopyExecutionStateStep (#35568) + + In the event that the target index does not exist when `CopyExecutionStateStep` + executes, this avoids a `NullPointerException` and provides a more helpful error + to the ILM user. + + Resolves #35567 + +commit 19001e71ccd7fa15e4d48e114bfc1e470c8c74ac +Author: debadair +Date: Thu Nov 15 08:22:13 2018 -0800 + + [DOCS] Updated release state for 7.0.0-alpha1 (#35596) + + * [DOCS] Updated release state for 7.0.0-alpha1 + + * Update Versions.asciidoc + +commit 1e60b282fc925be4d56d3706df7e8802e570e15a +Author: Tim Vernum +Date: Fri Nov 16 02:14:58 2018 +1100 + + Grant .tasks access to kibana_system role (#35573) + + Kibana now uses the tasks API to manage automatic reindexing of the + .kibana index during upgrades. + + The implementation of the tasks API requires that + 1. the user executing the task can create & write to the ".tasks" index + 2. the user checking on the status of the task can read (Get) the + relevant document from the ".tasks" index + +commit 2d948a001eb7694d438910af96f6c5778fb58aca +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Thu Nov 15 14:57:17 2018 +0000 + + [HLRC][ML] Add ML delete model snapshot API (#35537) + + Relates to #29827 + +commit 68d9b56fcb47d9c813b3f65d7253ee9f3e5fc661 +Author: Armin Braun +Date: Thu Nov 15 14:24:04 2018 +0100 + + MINOR: Remove SNAPSHOT Suffixes in Gradle (#35577) + + * MINOR: Remove SNAPSHOT Suffixes in Gradle + + * The snapshot qualifiers here are redundant and 6.5.0 is released anyway + +commit e6f4425e01afda926f4139cfab4335f2a499892c +Author: Armin Braun +Date: Thu Nov 15 14:23:28 2018 +0100 + + REST-Tests: Use Dedicated AntBuilders (#35576) + + * REST-Tests: Use Dedicated AntBuilders + + * Use dedicated AntBuilder everywhere since AntBuilder is not threadsafe + * Closes #33778 + +commit eaf010cd737dc9b0cd4aac9c800e732d37a1ed8f +Author: Andrei Stefan +Date: Thu Nov 15 14:33:47 2018 +0200 + + SQL: clear the cursor if nested inner hits are enough to fulfill the query required limits (#35398) + +commit bc5b1afb55c3b9a75c8d3f69c7832765918b439c +Author: Christoph Büscher +Date: Thu Nov 15 10:50:48 2018 +0100 + + HLRC: Add parameters to stopRollupJob API (#35545) + + With #34811 the API for stopping rollup jobs got two new url parameters + "wait_for_completion" and "timeout". This change adds these to the HLRC APIs as + well. + + Relates to #34811 + +commit 8e2c84ad8b84047ef6ac7a4eae0f3de68745b658 +Author: Luca Cavanna +Date: Thu Nov 15 09:59:27 2018 +0100 + + Clean up XPackInfoResponse class and related tests (#35547) + + Response classes in Elasticsearch (and xpack) only need to implement ToXContent, which is needed to print their output put in the REST layer and return the response in json (or others) format. On the other hand, response classes that are added to the high-level REST client, need to do the opposite: parse xcontent and create a new object based on that. + + This commit removes the parsing code from the XPackInfoResponse server variant, and the toXContent portion from the corresponding client variant. It also removes a client specific test class that looks redundant now that we have a single test class for both classes. + +commit 928ab6a5b1efddd6c3d8f24e46f5bd4011784dcb +Author: David Turner +Date: Thu Nov 15 08:25:09 2018 +0000 + + Suppress CachedTimeThread in hot threads output (#35558) + + In #23175 we renamed `ThreadPool$EstimatedTimeThread` to + `ThreadPool$CachedTimeThread` but did not update the corresponding entry in + `HotThreads#isIdleThread`. This commit addresses this. + +commit c9b4ef0dfd11ce5e0ed3fa27c314cc1ff18c724b +Author: Tanguy Leroux +Date: Thu Nov 15 09:24:40 2018 +0100 + + Use RunOnce when appropriate (#35553) + + This pull request replaces some blocks of code that must be run once + and that are currently based on AtomicBoolean by the convient RunOnce + class added in #35489. + +commit 135c3f0f077003b99dc8f38e64036da023347ae5 +Merge: 045fdd0d3b3 dc1821c7075 +Author: David Turner +Date: Thu Nov 15 08:24:26 2018 +0000 + + Merge branch 'master' into zen2 + +commit cb8bdeae68ad946990172acfec2ecffe27900d7b +Author: Clinton Gormley +Date: Thu Nov 15 09:18:33 2018 +0100 + + Fixed bad link in ingest-geo-point + +commit 5cd9b518acd80be4139f2492fb34c23aaba7c294 +Author: Tanguy Leroux +Date: Thu Nov 15 09:18:32 2018 +0100 + + Fix usages of XContentParserUtils.ensureExpectedToken() (#35530) + + Some usages of XContentParserUtils.ensureExpectedToken() are inverting + the expected and actual tokens, resulting in wrong error messages. + +commit dc1821c70757e8e568ad738d45b8690ef9cd0c7d +Author: Tal Levy +Date: Wed Nov 14 20:05:45 2018 -0800 + + explain geo_point mapping in geoip-processor (#29114) + + simple docs change to add missing mapping explanation. Users may not be aware this is + a prerequisite for doing geo-queries on this enriched data. + +commit 95a09ab2d686d7ed4c98dc5f19b6aa810d95c748 +Author: Tal Levy +Date: Wed Nov 14 16:45:06 2018 -0800 + + [ILM] TEST: skip checking _ilm/explain response b/c flaky timing (#35562) + + The docs are not resilient to timing issues where the ILM metadata is not set on newly + created indices, so we shouldn't be so strict on the returned response + +commit 91a38b8fe0cf824356799507fbfc1826234edf44 +Author: Deb Adair +Date: Wed Nov 14 16:06:51 2018 -0800 + + [DOCS] Fixed bad cross doc link. + +commit faa9523d1999f9c9d2072a57d7d383d47f779588 +Author: Jay Modi +Date: Wed Nov 14 17:04:33 2018 -0700 + + Remove deprecated constructor from failure handler (#35565) + + The DefaultAuthenticationFailureHandler has a deprecated constructor + that was present to prevent a breaking change to custom realm plugin + authors in 6.x. This commit removes the constructor and its uses. + +commit 38d6bb27f578aa6872964186bcfdc2b3584151c9 +Author: debadair +Date: Wed Nov 14 15:23:28 2018 -0800 + + [DOCS] Added link to standalone ODBC driver doc. (#35526) + +commit 5d0544d732e99eec093b9dac52e2f0f59c3598c8 +Author: Tal Levy +Date: Wed Nov 14 14:08:31 2018 -0800 + + update move-to-step spec with proper docs and extra warning to clients (#35477) + +commit f70d4470f7e3c220bc4c9e7338331eac373e2b4b +Author: Tal Levy +Date: Wed Nov 14 13:58:08 2018 -0800 + + [ILM] getting started documentation (#35379) + +commit 8ea999e4898a67016c68a118bf27d2a63f183064 +Author: Lee Hinman +Date: Wed Nov 14 14:40:05 2018 -0700 + + Include stack trace with ILM error in explain output (#35512) + + This changes the stacktrace to be included with the ILM explain error when the + index is an on ERROR step. + + Before: + + ```json + { + "indices" : { + "foo" : { + "index" : "foo", + "managed" : true, + "policy" : "bad", + "lifecycle_date_millis" : 1542131670601, + "phase" : "warm", + "phase_time_millis" : 1542131676335, + "action" : "shrink", + "action_time_millis" : 1542131676335, + "step" : "ERROR", + "step_time_millis" : 1542131676451, + "failed_step" : "shrink", + "step_info" : { + "type" : "illegal_argument_exception", + "reason" : "the number of target shards [13] must be less that the number of source shards [2]" + }, + "phase_execution" : { + "policy" : "bad", + "phase_definition" : { + "min_age" : "5s", + "actions" : { + "shrink" : { + "number_of_shards" : 13 + } + } + }, + "version" : 1, + "modified_date_in_millis" : 1542131669839 + } + } + } + } + ``` + + After + + ``` + { + "indices" : { + "foo" : { + "index" : "foo", + "managed" : true, + "policy" : "bad", + "lifecycle_date_millis" : 1542131670601, + "phase" : "warm", + "phase_time_millis" : 1542131676335, + "action" : "shrink", + "action_time_millis" : 1542131676335, + "step" : "ERROR", + "step_time_millis" : 1542131676451, + "failed_step" : "shrink", + "step_info" : { + "type" : "illegal_argument_exception", + "reason" : "the number of target shards [13] must be less that the number of source shards [2]", + "stack_trace" : "java.lang.IllegalArgumentException: the number of target shards [13] must be less that the number of source shards [2]\n\tat org.elasticsearch.cluster.metadata.IndexMetaData.selectShrinkShards(IndexMetaData.java:1509)\n\tat org.elasticsearch.action.admin.indices.shrink.TransportResizeAction.prepareCreateIndexRequest(TransportResizeAction.java:146)\n\tat org.elasticsearch.action.admin.indices.shrink.TransportResizeAction$1.onResponse(TransportResizeAction.java:104)\n\tat org.elasticsearch.action.admin.indices.shrink.TransportResizeAction$1.onResponse(TransportResizeAction.java:101)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:64)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:60)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onCompletion(TransportBroadcastByNodeAction.java:383)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onNodeResponse(TransportBroadcastByNodeAction.java:352)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:324)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:314)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1117)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1198)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1178)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:54)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:417)\n\tat org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:391)\n\tat org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)\n\tat org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)\n\tat org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:714)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:726)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\n" + }, + "phase_execution" : { + "policy" : "bad", + "phase_definition" : { + "min_age" : "5s", + "actions" : { + "shrink" : { + "number_of_shards" : 13 + } + } + }, + "version" : 1, + "modified_date_in_millis" : 1542131669839 + } + } + } + } + ``` + + Resolves #35498 + +commit fc774a377671d9f633513b3f8d954b237b377178 +Author: Hendrik Muhs +Date: Wed Nov 14 21:08:21 2018 +0100 + + add ES 6.5.1 (on master) (#35549) + + add ES 6.5.1 + +commit 30a713cc3f8a890090153e76a2afaa1080344150 +Author: Tal Levy +Date: Wed Nov 14 09:23:50 2018 -0800 + + [ILM] add docs explaining high-level concepts (#35322) + + add docs explaining high-level concepts + +commit 803eccec11c66ab1c0b99d7819ddfb928ee46200 +Author: Benjamin Trent +Date: Wed Nov 14 11:13:11 2018 -0600 + + HLRC: Adding ML Update Filter API (#35522) + + * HLRC: Adding ml get filters api + + * HLRC: Adding ML Update Filter API + +commit c8c8ce2374e126db8175222d997cbd5dcbb97fe5 +Author: Tanguy Leroux +Date: Wed Nov 14 17:33:04 2018 +0100 + + Extract RunOnce into a dedicated class (#35489) + + This commit extracts the static inner class RunOnce from + WorkerBulkByScrollTaskState so that it can be reused at + other places. + +commit 98e075227bcd0998542d805f08a04368c809170d +Author: Thiago Souza +Date: Wed Nov 14 14:27:18 2018 -0200 + + [Doc] Correct jdbc url example (#35513) + + Adds a missing `?` to separate the url parameters. + +commit a3cb74190cf599f8abe74b5961964609ee9ab8cf +Author: Alpar Torok +Date: Wed Nov 14 16:51:54 2018 +0200 + + Fix flaky gradle integration test + +commit c7ea0343d235c6af84f31c6293b253cd593bbc33 +Author: Zachary Tong +Date: Wed Nov 14 09:56:45 2018 -0500 + + [Rollup] Update versions after backport + + Both serialization and yaml test skips + +commit 5b7446bb5fbcb8776ae147aafea44763250cc97c +Author: Tanguy Leroux +Date: Wed Nov 14 14:04:30 2018 +0100 + + Add Delete Privileges API to HLRC (#35454) + + This commit adds the Delete Privileges API to the high level REST + client. + + Related to #29827 + +commit e7b7d52a6a3fc1bf4d32191bd5d7db03c4aa1ff4 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Wed Nov 14 13:03:04 2018 +0000 + + [HLRC][ML] Add ML get model snapshots API (#35487) + + + Relates #29827 + +commit 045fdd0d3b32057dbcee20315232d34d71897d4c +Merge: 229637fd7e9 4cfdb0609e4 +Author: Andrey Ershov +Date: Wed Nov 14 15:36:05 2018 +0300 + + Merge master into zen2 + +commit 4cfdb0609e4efb81510f590fab95033e0904965f +Author: Yannick Welsch +Date: Wed Nov 14 13:24:56 2018 +0100 + + Adapt InternalCluster#fullRestart to call onNodeStopped when all nodes are stopped (#35494) + + Refactors and simplifies the logic around stopping nodes, making sure that for a full cluster restart + onNodeStopped is only called after the nodes are actually all stopped (and in particular not while + starting up some nodes again). This change also ensures that a closed node client is not being used + anymore (which required a small change to a test). + + Relates to #35049 + +commit 212c20270961f9914f0a3bbde6faaf2635e67024 +Author: Alpar Torok +Date: Wed Nov 14 14:14:03 2018 +0200 + + Remove hard coded version string in tests + + - The current version was hard coded in the test, causing it to fail as + we removed the qualifier + - also applied the base plugin to the root project to get the `clean` + task to work as expected. This was preventing the failure from + reproducing locally. + +commit 72504c2512de41a742f39ef2b570a80999a5cd43 +Author: Jim Ferenczi +Date: Wed Nov 14 10:50:31 2018 +0100 + + Do not recommend to use the _id field in search_after docs (#35370) + + The documentation of `search_after` recommends to use the `_id` + field as a tiebreaker for the sort without warning against + the additional memory required. This change changes the recommandation + to use a copy of the `_id` field with doc_values enabled. + +commit 0ce4649e889d08157c51929518c9298bfe50eacd +Author: Alpar Torok +Date: Wed Nov 14 11:22:00 2018 +0200 + + Manage dependencies for test clusters (#35304) + + * Manage dependencies for test clusters + + Create a configuration and add the distribution to it automatically. + A task is created and added as a dependency to any task that uses a test + cluster. + The task extracts all the zip archives ( only zip support for now ) + in the configuration. + We do this only once because most tests mostly use the same distribution + and thus we can avoid extracting it multiple times. + With this we will be able to start the node from the same files which + will most of the time live in OS caches or COW if the + configuration requires it. + +commit bbe50e7a86c730cf799be2d74933869e532de73d +Author: Tanguy Leroux +Date: Wed Nov 14 10:12:25 2018 +0100 + + Remove LoggingRunnable class (#35486) + + This commit removes the unused LoggingRunnable class. + +commit dc0e05f35ae0918a2971310edb8bfefa3cf45ffe +Author: Alpar Torok +Date: Wed Nov 14 11:11:46 2018 +0200 + + Fix Eclipse configuration (#35452) + +commit b49f461bf1e1a6adda090b956ef1dc56fcbf4dec +Author: Alpar Torok +Date: Wed Nov 14 10:50:08 2018 +0200 + + Swithc to jcenter and hope to have fewer network failures in builds (#35427) + + * Swithc to jcenter + + Jcenter suposedly operates on a CDN. + It should be faster and more reliable. + It's the default in Andorid Studio currently + ( it switched from mavenCentral ). + + This change is safe because jcenter is a superset of mavenCentral. + + * update comment + +commit 31567cefb4dc46c146c65c58204af825b491b743 +Author: Tanguy Leroux +Date: Wed Nov 14 09:43:55 2018 +0100 + + [RCI] Check blocks while having index shard permit in TransportReplicationAction (#35332) + + Today, the TransportReplicationAction checks the global level blocks and + the index level blocks before routing the operation to the primary, in the + ReroutePhase, and it happens at the very beginning of the transport + replication action execution. For the upcoming rework of the Close Index + API and in order to deal with primary relocation, we'll need to also check + for blocks before executing the operation on the primary (while holding a + permit) but before routing to the new primary. + + This pull request change the AsyncPrimaryAction so that it checks for + replication action's blocks before executing the operation locally or before + routing the primary action to the newly primary shard. The check is done + while holding a PrimaryShardReference. + + Related to #33888 + +commit 5c84708ee5d84f72a25d082f92db69a8bbed0346 +Author: Hendrik Muhs +Date: Wed Nov 14 08:23:49 2018 +0100 + + test: expose error message on failure + +commit 231f6c1595a203656c6048b58de371966c7f99be +Author: Tim Vernum +Date: Wed Nov 14 16:46:31 2018 +1100 + + Formal support for "password_hash" in Put User (#35242) + + For some time, the PutUser REST API has supported storing a pre-hashed + password for a user. The change adds validation and tests around that + feature so that it can be documented & officially supported. + + It also prevents the request from containing both a "password" and a "password_hash". + +commit d8b1c23e1dd83f8c35e0e4d6cb42675c834f4a0a +Author: Christoph Büscher +Date: Wed Nov 14 05:58:05 2018 +0100 + + Remove Comparable interface from ScoreAccessor (#35519) + + The way ScoreAccessor implements `compareTo()` is problematic because it doesn't + completely follow the Comparable contract, specificaly symmetry (if x is a + ScoreAccessor and y any Number then x.comparTo(y) works, but y.compareTo(x) + generally does not even compile). Fortunately we don't seem to use the fact that + ScoreAccessor is a Comparable anywhere, so we can simply remove it. + +commit 603d1a470fe080d33e4def82b01c249c36bd8a2a +Author: Christoph Büscher +Date: Wed Nov 14 05:40:42 2018 +0100 + + Add stop rollup job support to HL REST Client (#34702) + + This change adds support for stoping a rollup job to the High Level REST Client. + + Relates to #29827 + +commit 229637fd7e9f26e3345b955e3a14148664d9ae60 +Author: David Turner +Date: Tue Nov 13 22:30:36 2018 +0000 + + [Zen2] Remove duplicate discovered peers (#35505) + + Today the `PeerFinder` probes each address it obtains, identifies the node to + which it just connected, and then returns all such nodes. However, this can + lead to duplicates if a node manages to connect to another node via two + distinct addresses. This causes bootstrapping to fail since + `BootstrapConfiguration#resolve` forbids duplicates. + + This change alters the behaviour of the `PeerFinder` to remove duplicates in + this situation. + +commit c346a0f0279e874dd3160c78f0135855be0a3c4a +Author: Zachary Tong +Date: Tue Nov 13 16:37:17 2018 -0500 + + [Rollup] Add `wait_for_completion` option to StopRollupJob API (#34811) + + This adds a `wait_for_completion` flag which allows the user to block + the Stop API until the task has actually moved to a stopped state, + instead of returning immediately. If the flag is set, a `timeout` parameter + can be specified to determine how long (at max) to block the API + call. If unspecified, the timeout is 30s. + + If the timeout is exceeded before the job moves to STOPPED, a + timeout exception is thrown. Note: this is just signifying that the API + call itself timed out. The job will remain in STOPPING and evenutally + flip over to STOPPED in the background. + + If the user asks the API to block, we move over the the generic + threadpool so that we don't hold up a networking thread. + +commit 634bc9d4ea5d8873c1c7148477ad8072bebfc667 +Author: jaymode +Date: Tue Nov 13 14:35:05 2018 -0700 + + mute TasksClientDocumentationIT testCancelTasks + + This commit mutes this test, which has failed twice in CI today. + + See #35514 + +commit b9eb5f7b63918376f0c3b3c6b7f6a2d39bed67a7 +Author: Benjamin Trent +Date: Tue Nov 13 14:53:32 2018 -0600 + + HLRC: Adding ml get filters api (#35502) + + * HLRC: Adding ml get filters api + + * refactoring setId name + +commit c934fb087ac95fd134654fe0e5066ad33e2d6829 +Author: Colin Goodheart-Smithe +Date: Tue Nov 13 20:40:53 2018 +0000 + + Adds HLRC docs for put lifecycle policy (#35457) + + * Adds HLRC docs for put lifecycle policy + + * Adds link to docs in client javadocs + + * Fixes checkstyle + + * Make the documentation use the right ack response + +commit 9728119b82dff639d24b648841d85ad7c66f4fc9 +Author: Vladimir Dolzhenko +Date: Tue Nov 13 20:57:50 2018 +0100 + + [CI] AllocationIdIT testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStalePrimary failure + + Closes #35504 + +commit bc799e4a6fb8bf2e294cd61bb6b6c4e7a4018248 +Author: Julie Tibshirani +Date: Tue Nov 13 11:56:01 2018 -0800 + + Ignore warnings related to types deprecation in REST tests. (#35395) + +commit 8e40a2bbe2978aa3e630a4f334426e6dd162239a +Author: David Turner +Date: Tue Nov 13 19:32:32 2018 +0000 + + [Zen2] Introduce vote withdrawal (#35446) + + If shutting down half or more of the master-eligible nodes, their votes must + first be explicitly withdrawn to ensure that the cluster doesn't lose its + quorum. This works via _voting tombstones_, stored in the cluster state, which + tell the reconfigurator to remove nodes from the voting configuration. + + This change introduces voting tombstones to the cluster state, together with + transport APIs for adding and removing them, and makes use of these APIs in + `InternalTestCluster` to support tests which remove at least half of the + master-eligible nodes at once (e.g. shrinking from two master-eligible nodes to + one). + +commit 6cb2aef94374af1ddff0575e89a53656565da0f7 +Author: Lee Hinman +Date: Tue Nov 13 12:19:22 2018 -0700 + + [ILM] Add documentation for error handling in ILM (#35426) + + * [ILM] Add documentation for error handling in ILM + + This adds some initial documentation for error handling and retrying failed + steps for index lifecycle management + +commit 3db671e12ac0d049c6953107466bad5ad3cbe429 +Author: Armin Braun +Date: Tue Nov 13 20:26:42 2018 +0200 + + MINOR: Remove Hack in TestCluster Task (#35485) + + * The logic that splits by `","` isn't necessary anymore since #35375 removed the hack that set the comma separated list for the unicast host + +commit 2ac4ef7a696b84b733eb6a01eb8031440f0c2d78 +Author: lcawl +Date: Tue Nov 13 10:04:26 2018 -0800 + + [DOCS] Fixes broken link + +commit 8b2f22ee075836280a81486ff31210f383e2c775 +Author: Jason Tedor +Date: Tue Nov 13 12:15:37 2018 -0500 + + Introduce CCR overview (#35436) + + This commit introduces a basic overview for cross-cluster replication to + the docs. + + Co-authored-by: "lcawl " + +commit 73a563c915201239cf9b700dd362e08ebadf9630 +Author: Colin Goodheart-Smithe +Date: Tue Nov 13 17:02:56 2018 +0000 + + Converts ILM HLRC APIs to use client Ack response (#35492) + +commit 0e1a12122c4525a514aba058f3c7cdadc61911aa +Merge: fbd3cab4107 cbb16d5ffcd +Author: David Turner +Date: Tue Nov 13 15:25:35 2018 +0000 + + Merge branch 'master' into zen2 + +commit fbd3cab41078b47bfd468210f42a0719649fde11 +Author: David Turner +Date: Tue Nov 13 15:20:49 2018 +0000 + + [Zen2] Remove AbstractComponent usage (#35483) + + AbstractComponent was deprecated in #35140 and is looking like it will be + removed at some point by #34888. Today all it does is provide a logger. This + change removes the usages of AbstractComponent that live solely in the zen2 + feature branch to avoid some future merge pain, and replaces it where necessary + with some directly-created loggers. + +commit 0b7d18d7330e9e9cf4274bdd4be9c487d3ddd72e +Author: Luca Cavanna +Date: Tue Nov 13 15:52:59 2018 +0100 + + HLRC: add support for get license basic/trial status API (#33176) + + Relates to #29827 + +commit a76ac5729d81208c9a35db1dcfb5c84eddf1ba21 +Author: Igor Motov +Date: Tue Nov 13 04:47:35 2018 -1000 + + Rest HL client: Add watcher stats API (#35185) + + Relates to #29827 + +commit 4b5fbada9fae5b59995c0f432498186e06b557d7 +Author: Jason Tedor +Date: Tue Nov 13 09:14:26 2018 -0500 + + Fix reference to get follower stats API + + This commit fixes a reference to the CCR get follower stats API in the + CCR getting started docs. + +commit fd4cd804969d2a5a3f595cbbfe796a4f51bf3993 +Author: Przemyslaw Gomulka +Date: Tue Nov 13 15:01:53 2018 +0100 + + HLRC: migration api - upgrade (#34898) + + Implement high level client for migration upgrade API. It should wrap + RestHighLevelClient and expose high level IndexUpgradeRequest (new), + IndexTaskResponse for submissions with wait_for_completion=false and + BulkByScrollResponse (already used) objects. + + refers: #29827 + +commit 0487181d0f2d9fdfda8d0d543219c62a00c379d9 +Author: Martijn van Groningen +Date: Tue Nov 13 14:24:42 2018 +0100 + + [TEST] Force flush to ensure multiple segments. + + Relates to #35333 + +commit 3229dfc4de1b6464f5656bad145e13eec48cf17c +Author: Simon Willnauer +Date: Tue Nov 13 14:53:55 2018 +0100 + + Allow efficient can_match phases on frozen indices (#35431) + + This change adds a special caching reader that caches all relevant + values for a range query to rewrite correctly in a can_match phase + without actually opening the underlying directory reader. This + allows frozen indices to be filtered with can_match and in-turn + searched with wildcards in a efficient way since it allows us to + exclude shards that won't match based on their date-ranges without + opening their directory readers. + + Relates to #34352 + Depends on #34357 + +commit e81671dd4559bfe6a85e4e8258b3caf0ab8ac933 +Author: Martijn van Groningen +Date: Tue Nov 13 13:22:49 2018 +0100 + + [HLRC] Added support for CCR Put Follow API (#35409) + + This change also adds documentation for the Put Follow API and + adds a CCR HLRC integration test. + +commit 7919b66e6730efccc65a1579e1c0fa10b09fb8a7 +Author: Armin Braun +Date: Tue Nov 13 14:19:05 2018 +0200 + + DISCOVERY: 0s Initial State Timeout in Tests (#35481) + + * DISCOVERY: 0s Initial State Timeout in Tests + + * Don't wait for initial state even with a single node, otherwise the loop writing the discovery file causes that single node to wait + for its own transport.ports file for 30s. + * Closes #35456 + +commit cbb16d5ffcd03573c9e39d634968d3b42c7f591b +Author: Jason Tedor +Date: Tue Nov 13 06:48:56 2018 -0500 + + Remove extraneous spaces from CCR getting started + + This commit removes some extraneous spaces that were committed to the + cross-cluster replication getting started docs. + +commit 92fef40dbfc2b2e63147ac6ae1b30f707de42043 +Author: Jason Tedor +Date: Tue Nov 13 06:45:00 2018 -0500 + + Introduce CCR getting started guide (#35434) + + This commit introduces a basic getting started guide for cross-cluster + replication to the docs. + + Co-authored-by: "lcawl " + +commit 76cefb06e31967c6db2bdb827871e8d40741473a (tag: v7.0.0-alpha1) +Author: Lukáš Vlček +Date: Tue Nov 13 10:57:12 2018 +0100 + + Correct typo in BuildPlugin exception message (#35458) + +commit 0a6614a03a2299cba3c504f6019dd84b1d0d4993 +Author: Christoph Büscher +Date: Tue Nov 13 10:34:29 2018 +0100 + + Correct implemented interface of ParsedReverseNested (#35455) + + The ParsedReverseNested implementation should implement the ReverseNested + interface and not the Nested interface. Although this is an empty marker + interface it is confusing and can lead to casting errors. Also adding a test to + check that both ParsedNested and ParsedReverseNested implement the correct + interface. + + Closes #35449 + +commit 09965cb370e29bc2ef5677af07626c5d37e7eaa6 +Author: David Roberts +Date: Tue Nov 13 08:49:24 2018 +0000 + + [ML] Fix find_file_structure NPE with should_trim_fields (#35465) + + The NPE would occur if should_trim_field was overridden to + true and any field value was completely blank. This change + defends against this situation. + + Fixes #35462 + +commit a18b599d645b73f1325004e78558d4071b6a344c +Author: Jason Tedor +Date: Mon Nov 12 19:31:12 2018 -0500 + + Handle OS pretty name on old OS without OS release (#35453) + + Some very old ancient versions of Linux do not have /etc/os-release. For + example, old Red Hat-like OS. This commit adds a fallback for handling + pretty name for these OS. + +commit 71cfb730f6cc92595b465db6e1be5677325bc0a2 +Author: Tim Brooks +Date: Mon Nov 12 16:07:42 2018 -0700 + + Register remote cluster compress setting (#35464) + + This is a follow up to #35357. That commit failed to register the new + cluster.remote.cluster_name.transport.compress setting with + `ClusterSettings`. This commit fixes that. + +commit 9aeaceac4b8330f412da2ecc4a006d771a524a79 +Author: Lisa Cawley +Date: Mon Nov 12 13:08:57 2018 -0800 + + [DOCS] Clarify results_index_name description (#35463) + +commit 711d703ffce05261d8ec9ff3ebd3d10fa5eebc7c +Author: Zachary Tong +Date: Mon Nov 12 15:31:16 2018 -0500 + + [TEST] add missing doc tests for HLRC GetRollupIndexCaps API + +commit 2da239fb5eb82ec397ea195b59c18542ee6d3f11 +Author: Zachary Tong +Date: Mon Nov 12 15:05:03 2018 -0500 + + [HLRC] Add GetRollupIndexCaps API (#35102) + + Also refactors the caps response tests a bit to share the same + abstract class to reduce duplication of test code + +commit e7896bcefc1f4bd3927c4926ad04dd5fff68df13 +Author: Igor Motov +Date: Mon Nov 12 09:40:04 2018 -1000 + + Geo: enables coerce support in WKT polygon parser (#35414) + + WKT parser now automatically closes open polygons similar to GeoJSON + parser if coerce flag in mapping is set to true. + + Closes to #35059 + +commit 16cbbab7b7e2bfe7665ee5e471772b634a0b5001 +Author: Tal Levy +Date: Mon Nov 12 11:32:59 2018 -0800 + + [ILM] fix retry so it picks up latest policy and executes async action (#35406) + + Before, moving to a failed step would only change the step info + to be that of the failed step. This means two things. + + 1. Async Steps would never be triggered to execute + 2. If there are inherent problems with the action definition that can + be fixed with a policy update, these changes were not being reflected + by the new execution info. + + Changes now + + 1. Async steps are executed after the move to the failed step in cluster state + 2. the lifecycle execution info's phase definition is updated from the current + latest policy definition, even though the index isn't moving to a new phase. + + Closes #35397. + +commit 40ca62c298a5852a300bfc6c4bd81ba70e907285 +Author: Jason Tedor +Date: Mon Nov 12 14:27:57 2018 -0500 + + Address handling of OS pretty name on some OS (#35451) + + Some OS (e.g., Oracle Linux Server 6.9) have a trailing space at the end + of the PRETTY_NAME line in /etc/os-release. This commit addresses this + by accounting for this trailing space when extracting the pretty name. + +commit d2ff01af13df1aab31d43e27571496a31c384723 +Author: Yannick Welsch +Date: Mon Nov 12 19:31:10 2018 +0100 + + Zen2: Add basic Zen1 transport-level BWC (#35443) + + Implements serialization compatibility between Zen1 and Zen2 transport action, allowing a Zen1 node to join a fully formed Zen2 cluster and vice-versa. + +commit 64e5c254c1b5ed13217b0212abde53482bfe3e13 +Author: markharwood +Date: Mon Nov 12 17:07:18 2018 +0000 + + HLRC support for getTask (#35166) + + Given a GetTaskRequest the API returns an Optional which is empty in the case of 404s or returns a TaskInfo object if found. + Added Helper methods in RestHighLevelClient for returning empty Optionals when hitting 404s + +commit 2591f66a33d1d2c558ad97c65fb60b092b708678 +Author: Nick Knize +Date: Mon Nov 12 10:48:27 2018 -0600 + + upgrade to lucene-8.0.0-snapshot-6d9c714052 (#35428) + +commit 326b2388523789d1819b70eb0f6feded746c81fe +Author: Jason Tedor +Date: Mon Nov 12 10:28:35 2018 -0500 + + Add docs for CCR stats API (#35439) + + This commit adds API docs for the CCR stats endpoint. + +commit 3859d2166194efa5f456ada2639a25188f57d53c +Author: Jason Tedor +Date: Mon Nov 12 10:27:12 2018 -0500 + + Fix the names of CCR stats endpoints in usage API (#35438) + + This commit fixes the names of the CCR stats endpoints reported in the + usage API. + +commit ca24d9a07edabaafeb0b04d94bdff49bb5d975ee +Author: Alpar Torok +Date: Mon Nov 12 14:43:42 2018 +0200 + + Switch to default to no build qualifier (#35415) + +commit c8e685eb3462674dd213850d4a2b8fba785ecb2e +Author: Peter Dyson +Date: Mon Nov 12 22:04:33 2018 +1000 + + Clarify S3 repository storage class parameter (#35400) + + Today it is unclear that the `storage_class` parameter to an S3 repository only + affects new objects and does not rewrite any existing objects. This commit + clarifies this point. + +commit fe29b18c26c40ef33ea13d2696128224f2275a58 +Author: Yannick Welsch +Date: Mon Nov 12 11:05:11 2018 +0100 + + Fix compilation + +commit 55ff96b43f4fbb9bcdebb69a6764b4760b80d626 +Author: Marios Trivyzas +Date: Mon Nov 12 10:42:27 2018 +0100 + + SQL: Fix query translation for scripted queries (#35408) + + IsNull, IsNotNull and Not where generating wrong queries + as the check to generate ScriptQuery was missing. + + Fixes: #35232 + +commit ef10461caf26a89b0f39ff1e34b3f32a8ab708c0 +Author: Martijn van Groningen +Date: Mon Nov 12 10:08:33 2018 +0100 + + [TEST] Instead of ignoring the ccr downgrade to basic license qa test + avoid the assertions that check the log files, because that does not work on Windows. + The rest of the test is still useful and should work on Windows CI. + + Currently on Windows CI this qa module fails because there is just one test and + that test si ignored if OS is Windows. + +commit 4e6c58c942b9aaab5437e6040acab0e505f8e497 +Merge: f69a5c9b3c1 09cac321e72 +Author: Yannick Welsch +Date: Mon Nov 12 10:03:59 2018 +0100 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 09cac321e72e88d0ec81ff8d57c98be815cd6408 +Author: Christoph Büscher +Date: Mon Nov 12 10:02:41 2018 +0100 + + Upgrade to Joda 2.10.1 (#35410) + + This version contains a bugfix that allows us to reenable one of our muted tests + in DateTimeUnitTests. + + Closes #33749 + +commit 6050deb6f0791f7d2c2534ca13df7779cfdd9521 +Author: Albert Zaharovits +Date: Sun Nov 11 17:13:25 2018 +0200 + + HLRest: model role and privileges (#35128) + + The following "user privileges" objects have been created + on the client side: + + * ApplicationResourcePrivileges + * IndicesPrivileges + * GlobalOperationPrivilege + * GlobalPrivileges + + as well as the aggregating `Role` entity. + +commit eb8f3462042693b950d8200a4d740f07641b7be4 +Author: Jason Tedor +Date: Sat Nov 10 15:45:38 2018 -0500 + + Fix beta label on CCR index page + + This commit fixes the location of the beta label on the CCR index page. + +commit d84ff3765d97e64a5f6d3fed454cf2c1e5014bfd +Author: Jason Tedor +Date: Sat Nov 10 14:37:14 2018 -0500 + + Mark CCR as beta in the docs + + This commit marks CCR as beta in the docs. This beta label is applied to + all CCR pages. + +commit 64df803af082bd7d09bdf0ddb4cd46bdc5c73cd3 +Author: Simon Willnauer +Date: Sat Nov 10 12:15:31 2018 +0100 + + Add a java level freeze/unfreeze API (#35353) + + This change adds a high level freeze API that allows to mark an + index as frozen and vice versa. Indices must be closed in order to + become frozen and an open but frozen index must be closed to be + defrosted. This change also adds a index.frozen setting to + mark frozen indices and integrates the frozen engine with the + SearchOperationListener that resets and releases the directory + reader after and before search phases. + + Relates to #34352 + Depends on #34357 + +commit ba478827ad29d25fc5f6623aa1ad86be81cc678f +Author: Tim Brooks +Date: Fri Nov 9 10:12:49 2018 -0700 + + Improve MockTcpTransport memory usage (#35402) + + The MockTcpTransport is not friendly in regards to memory usage. It must + allocate multiple byte arrays for every message. This improves the + memory situation by failing fast if the message is improperly formatted. + Additionally, it uses reusable big arrays for at least half of the + allocated byte arrays. + +commit 67f9e8fa234443eb754ed780e62e5deff18b67b0 +Author: Gordon Brown +Date: Fri Nov 9 10:11:26 2018 -0700 + + Enforce limitations on ILM policy names (#35104) + + Enforces restrictions on ILM policy names to ensure we don't accept + policy names the system can't handle, or may reserve for future use. + +commit ae2af20ae547f549c547d960e6dfc02e4d3c8cad +Author: Martijn van Groningen +Date: Fri Nov 9 17:43:43 2018 +0100 + + [CCR] Validate remote cluster license as part of put auto follow pattern api call (#35364) + + Validate remote cluster license as part of put auto follow pattern api call + in addition of validation that when auto follow coordinator starts auto + following indices in the leader cluster. + + Also added qa module that tests what happens to ccr after downgrading to basic license. + Existing active follow indices should remain to follow, + but the auto follow feature should not pickup new leader indices. + +commit f69a5c9b3c151378a5d72a7984a791e5e7804364 +Author: David Turner +Date: Fri Nov 9 15:50:11 2018 +0000 + + Fix compile error introduced by conflict in previous two commits + +commit f2d7c9494993bb1d715a6089c510cf2b49973192 +Author: Albert Zaharovits +Date: Fri Nov 9 16:58:10 2018 +0200 + + DOCS Add audit ignore settings to reference page (#35274) + + Adds the logfile audit ignore policy settings synopsis + to the Auditing Reference page. + +commit 807ce10f731806006c7d027783339d216792247c +Author: Martijn van Groningen +Date: Fri Nov 9 15:40:15 2018 +0100 + + [TEST] Increased timeout for verifying ccr monitoring. + +commit fba811fa3ab74dd6fe342c2908990f8c225d9fbb +Author: Martijn van Groningen +Date: Fri Nov 9 15:31:51 2018 +0100 + + [TEST] increased the number of index and delete ops to make it less likely that all ops exist as soft delete docs. + +commit 83152b3835d1ec3d5879638cc2661534a388e8ee +Author: Martijn van Groningen +Date: Fri Nov 9 14:24:27 2018 +0100 + + [CCR] Get all auto follow patterns and no auto follow metadata (#35381) + + Return empty response when querying all auto follow patterns, + but there is no auto follow metadata. + +commit 36da6e167112881e7cf0a2916c8989f37b4d7a08 +Author: Marios Trivyzas +Date: Fri Nov 9 11:32:38 2018 +0100 + + SQL: Introduce IsNull node to simplify expressions (#35206) + + Add `IsNull` node in parser to simplify expressions so that ` IS NULL` is + no longer translated internally to `NOT( IS NOT NULL)` + + Replace `IsNotNullProcessor` with `CheckNullProcessor` to encapsulate both + isNull and isNotNull functionality. + + Closes: #34876 + Fixes: #35171 + +commit 529910a43c44d93e608f0ba3433ddad393af00bd +Author: Armin Braun +Date: Fri Nov 9 10:47:30 2018 +0100 + + DISCOVERY: Fix RollingUpgradeTests (#35375) + + * DISCOVERY: Fix RollingUpgradeTests + + * Don't manually manage min master nodes if not necessary + * Remove some dead code + * Allow for manually supplying list of seed nodes + * Closes #35178 + +commit 617f91bb0fadc25a1259175e16870c35304fcf58 +Author: Albert Zaharovits +Date: Fri Nov 9 11:38:45 2018 +0200 + + DOCS Auditing search queries (#35301) + + This documents how to include the search queries in the audit log. + + There is a catch, that even if enabling `emit_request_body`, which should + output queries included in request bodies, search queries were not output + because, implicitly, no REST layer audit event type was included. + + This folk knowledge is herein imprinted. + +commit 7054e289faab5e7cb0767fe79e4a4b3458b8d476 +Author: Jim Ferenczi +Date: Fri Nov 9 09:41:51 2018 +0100 + + Add trace log of the request for the query and fetch phases (#34479) + + This change adds a logger for the query and fetch phases that prints all requests + before their execution at the trace level. This will help debugging cases where an issue + occurs during the execution since only completed queries are logged by the slow logs. + +commit a9641960e1be101c0bcd4e9bbd28c3a2a2cabeb5 +Author: Tim Vernum +Date: Fri Nov 9 18:13:24 2018 +1100 + + Refactor realm tests to use single settings (#35362) + + Many realm tests were written to use separate setting objects for + "global settings" and "realm settings". + Since #30241 there is no distinction between these settings, so these + tests can be cleaned up to use a single Settings object. + +commit 66e3ca97246580212593b966085332fa4f3af76e +Author: Alpar Torok +Date: Fri Nov 9 08:04:56 2018 +0200 + + Revert "Mute test for #35361" + + This reverts commit f9154758e6aecc4a8373904d3189535b15e8beb8. + +commit b0acd709d7ec2f0493ac561731b51fd14dcfeeb9 +Author: Tim Vernum +Date: Fri Nov 9 16:57:49 2018 +1100 + + DOCS: Add SecurityExtension breaking change (#35358) + + We changed the way realm settings are defined, and this affects custom + realms in SecurityExtensions. This change adds those details to the + breaking changes docs. + + Relates: #30241 + +commit 02b4e28534d13436175111aad49500ae8367bb8f +Author: Armin Braun +Date: Fri Nov 9 05:07:26 2018 +0100 + + #31608 Add S3 Setting to Force Path Type Access (#34721) + + * SNAPSHOTS: Use Path Style Access in S3 + + * Use path style access pattern to fix #31608 + * closes #31608 + +commit bccc99c2be126e9beb9263ba0ad23d6b22dd2a20 +Author: Tim Brooks +Date: Thu Nov 8 18:04:48 2018 -0700 + + Fix TcpTransport compression test (#35396) + + This commit fixes an assertion in the TcpTransportTests compresssion + test. + +commit b485e69a5fa4eeabe36072da781935ddf4d59195 +Author: debadair +Date: Thu Nov 8 15:26:27 2018 -0800 + + ILM API ref edits (#35112) + + * [DOCS] ILM API Ref edits + + * [DOCS] Fixed endpoint for DELETE policy. + + * [DOCS] Removed comparison to setting index.lifecycle.name to null. + + * [DOCS] Fixed xrefs to explain API. + +commit c9a84fe361220c379f19e458ee8a9e49078247d4 +Author: Marios Trivyzas +Date: Thu Nov 8 23:32:59 2018 +0100 + + SQL: Handle null literal for AND and OR in `WHERE` (#35236) + + Change `nullable()` logic of AND and OR to false since in the Optimizer + we cannot fold to null as we might be handling and expression in the + SELECT clause. + + Introduce folding of null for AND and OR expressions in PruneFilter() + since we now know that we are in HAVING or WHERE clause and we + can fold `null` to `false` + + Fixes: #35088 + + Co-authored-by: Costin Leau + +commit c88b2d7e5a33459bd986d21ac3329192358deecb +Author: Lisa Cawley +Date: Thu Nov 8 13:53:09 2018 -0800 + + [DOCS] Cleans up monitoring configuration steps (#35111) + +commit 7e144b318ac34433ea971f2574723c2636c90e1b +Author: Mayya Sharipova +Date: Thu Nov 8 14:54:20 2018 -0500 + + Painless Context Doc: Add field context example (#35130) + + * Painless Context Doc: Add field context example + + relates to #34829 + +commit 93c2c604e57cae4ba093c7053dec604a33d89e36 +Author: Tim Brooks +Date: Thu Nov 8 10:37:59 2018 -0700 + + Move compression config to ConnectionProfile (#35357) + + This is related to #34483. It introduces a namespaced setting for + compression that allows users to configure compression on a per remote + cluster basis. The transport.tcp.compress remains as a fallback + setting. If transport.tcp.compress is set to true, then all requests + and responses are compressed. If it is set to false, only requests to + clusters based on the cluster.remote.cluster_name.transport.compress + setting are compressed. However, after this change regardless of any + local settings, responses will be compressed if the request that is + received was compressed. + +commit 5c2a5f2e37063f9a89bec0f16c8f87223dd734be +Author: Jason Tedor +Date: Thu Nov 8 12:24:10 2018 -0500 + + Adjust BWC version on OS pretty name + + This commit adjusts the BWC version the OS pretty name field on OsInfo + now that this field has been backported to the 6.x development branch. + +commit 730ec1ddfb6c93413f44547f0221460c8fdb8e32 +Author: Jason Tedor +Date: Thu Nov 8 12:16:58 2018 -0500 + + Add more detailed OS name on Linux (#35352) + + Today our OS information returned in node stats only returns a + high-level name of the OS (e.g., "Linux"). Yet, for some uses this is + too high-level and knowing at a finer level of granularity the + underlying OS can be useful. This commit extracts the pretty name on + Linux from /etc/os-release. This pretty name usually includes the Linux + vendor and the Linux vendor version number (e.g., Fedora 28). + +commit 85a8b517bd659e3b036492da4fac6de942e8f053 +Author: Albert Zaharovits +Date: Thu Nov 8 18:42:36 2018 +0200 + + HLRest: refactor put_user to utilize `User` object (#35188) + + This follows #33552 , when the `_authenticate` API added a new + `User` object for the API's response. This changes the `put_user` + API to also employ a `User` object in the request. + The User object changed slightly. + A bug with put_user only putting/updating enabled (but not disabled) + users has been fixed. + +commit c315ead0ac7b45f5dd1b7f813911ccbda3dff192 +Author: Yannick Welsch +Date: Thu Nov 8 17:16:09 2018 +0100 + + Zen2: Add diff-based publishing (#35290) + + Enables diff-based publishing, which is an optimization where only the changing parts of the cluster + state are published to the nodes in the cluster, falling back to full cluster state publishing if the + receiver does not have the previous cluster state. + +commit 6885a7cb0ff92f1c7dab3e4a0a00784c851eea2f +Author: David Turner +Date: Thu Nov 8 16:09:37 2018 +0000 + + Introduce transport API for cluster bootstrapping (#34961) + + - Introduces a transport API for bootstrapping a Zen2 cluster + - Introduces a transport API for requesting the set of nodes that a + master-eligible node has discovered and for waiting until this comprises the + expected number of nodes. + - Alters ESIntegTestCase to use these APIs when forming a cluster, rather than + injecting the initial configuration directly. + +commit 113af7996cb8a9fd84371834cccfe4a5384ab4e2 +Author: Christoph Büscher +Date: Thu Nov 8 17:04:40 2018 +0100 + + Make limit on number of expanded fields configurable (#35284) + + Currently we introduced a hard limit of 1024 to the number of fields a query can + be expanded to in #26541. Instead of using a hard limit, we should make this + configurable. This change removes the hard limit check and uses the existing + `max_clause_count` setting instead. + + Closes #34778 + +commit 55c6481b6bfc94b37f4af4783bdd15776ab0f8e7 +Author: Zachary Tong +Date: Thu Nov 8 11:02:30 2018 -0500 + + [Docs] Add painless context details for bucket_selector (#35162) + + Adds docs for the bucket_selector context, an example + and corresponding doc test + +commit ac1d8da2837f9c0fd377ca019f13daec128b1771 +Author: Zachary Tong +Date: Thu Nov 8 10:31:30 2018 -0500 + + [Test] Remove unused imports + + Checkstyle + +commit 433469d4d05a8cb222ff0602dad6fd1664b55b94 +Author: Hendrik Muhs +Date: Thu Nov 8 16:28:48 2018 +0100 + + [Rollup] improve handling of failures on first search (#35269) + + Improve error handling in the Indexer if an exception occurs during the very 1st retrieval (query execution) + +commit 07a69a528ba09642e3ea8b7b0c17c954157698b8 +Author: Martijn van Groningen +Date: Thu Nov 8 16:26:14 2018 +0100 + + [CCR] Rename leaderClient variables and parameters to remoteClient (#35368) + +commit 54b445d74bf6538cf2d0d08afbb61446dd0a3df2 +Author: Zachary Tong +Date: Thu Nov 8 10:23:23 2018 -0500 + + [Test] Remove obsolete job/cluster cleanup code + + Also makes sure the awaitBusy for job stoppage is checked, so + that we can fail if we timed out waiting for a job to stop. + + Closes #35295 + +commit d96202a2821eb6ab6e97fad86c710a5591f2f6de +Author: Jim Ferenczi +Date: Thu Nov 8 15:40:01 2018 +0100 + + [DOCS] Fix missing callouts + +commit 6980feddd2292bc61fb15bf572d235e4f9bbf6ad +Author: Daniel Mitterdorfer +Date: Thu Nov 8 15:33:24 2018 +0100 + + Remove unused class MemoryCircuitBreaker + + The class `MemoryCircuitBreaker` is unused so we remove all its traces + from the code base. + + Relates #35367 + +commit 20cf13d10ea284b3a7ac329907813de568f69c9f +Author: Daniel Mitterdorfer +Date: Thu Nov 8 15:32:47 2018 +0100 + + Use Gradle wrapper in client benchmark README + + With this commit we use the Gradle wrapper in documentation examples for the + client benchmarks instead of Gradle itself. + + Relates #35366 + +commit ca12de5841e4c7c1a3cc6868f0842e96c2f859fc +Author: Alpar Torok +Date: Thu Nov 8 13:14:59 2018 +0200 + + Mute test for #35295 + +commit 77789a733d5bb4a466f015f24da59f8d2a65fba3 +Merge: 7e356ac29ba d00b23c8b13 +Author: David Turner +Date: Thu Nov 8 13:38:18 2018 +0000 + + Merge branch 'master' into 2018-11-08-merge-master + +commit d351422215548f518e497f78df32210ae245e260 +Author: Dominik Stadler +Date: Thu Nov 8 14:13:00 2018 +0100 + + Add parent-aggregation to parent-join module (#34210) + + Add `parent` aggregation, a special single bucket aggregation that joins children documents to their parent. + +commit 025a0c82e5c6177f419fea35cba843a15e904060 +Author: Albert Zaharovits +Date: Thu Nov 8 14:06:47 2018 +0200 + + Remove deprecated audit settings (#35205) + + Removes `.prefix` deprecated settings for the logfile + auditing and also documents it in the migrate asciidoc. + +commit 848847d8c767c5d5849ef7a1086367adfc9bd1ab +Author: Russ Cam +Date: Thu Nov 8 22:43:08 2018 +1100 + + [Docs] Section header preceded by blank line (#34340) + +commit 518e0de0781b57c1bd3d2510392c48f48986a622 +Author: Alpar Torok +Date: Thu Nov 8 12:01:19 2018 +0200 + + Mute test #35365 + +commit d00b23c8b139bc4485f9f67205166500b6f24460 +Author: Jeff Hajewski +Date: Thu Nov 8 04:09:03 2018 -0600 + + Fixes fast vector highlighter docs per issue 24318. (#34190) + + The `fvh` highlighter does not support span queries. This fix updates + the docs to add a warning stating the lack of span query support for + `fvh`. + +commit 9494e046e75d752aac26c66c01d090d3b2980991 +Author: David Kyle +Date: Thu Nov 8 09:57:07 2018 +0000 + + [ML] Prevent notifications on deletion of a non existent job (#35337) + +commit 8a85251da0922a5168b1a99527799aa2d37904cf +Author: Martijn van Groningen +Date: Thu Nov 8 10:48:27 2018 +0100 + + [CCR] Auto follow Coordinator fetch cluster state in system context (#35120) + + Auto follow Coordinator should fetch the leader cluster state using system context. + +commit f9154758e6aecc4a8373904d3189535b15e8beb8 +Author: Alpar Torok +Date: Thu Nov 8 11:27:46 2018 +0200 + + Mute test for #35361 + +commit 14b811446fa5a60cd5b367cec1dcbe62dbae3f20 +Author: Christoph Büscher +Date: Thu Nov 8 10:22:25 2018 +0100 + + Preserve `date_histogram` format when aggregating on unmapped fields (#35254) + + Currently when aggregating on an unmapped date field (e.g. using a + date_histogram) we don't preserve the aggregations `format` setting but instead + use the default format. This can lead to loosing the aggregations `format` when + aggregating over several indices where some of them contain unmapped date fields + and are encountered first in the reduce phase. + + Related to #31760 + +commit 0b24c4f8e7278911712b833e7cec38fc0ba2efbb +Author: Tim Vernum +Date: Thu Nov 8 19:21:16 2018 +1100 + + Test: Mute failing SSL test + + Relates: https://github.com/elastic/elasticsearch/issues/35360 + +commit 891fdda68ee6b2c7437457d386c1c2626cc3ffe8 +Author: Jim Ferenczi +Date: Thu Nov 8 09:30:52 2018 +0100 + + Allow unmapped fields in composite aggregations (#35331) + + Today the `composite` aggregation throws an error if a source targets an + unmapped field and `missing_bucket` is set to false. Documents without a + value for a source cannot produce any bucket if `missing_bucket` is not + activated so the error is a shortcut to say that the response will be empty. + However this is not consistent with the `terms` aggregation which accepts + unmapped field by default even if the response is also guaranteed to be empty. + This commit removes this restriction, if a source contains an unmapped field + we now return an empty response (no buckets). + + Closes #35317 + +commit 1703a61fec098ce57749bf5ba932dbeb6560a387 +Author: Tanguy Leroux +Date: Thu Nov 8 09:23:33 2018 +0100 + + [RCI] Add IndexShardOperationPermits.asyncBlockOperations(ActionListener) (#34902) + + The current implementation of asyncBlockOperations() can be used to + execute some code once all indexing operations permits have been acquired, + then releases all permits immediately after the code execution. This + immediate release is not suitable for treatments that need to keep all + permits over multiple execution steps. + + This commit adds a new asyncBlockOperations() that exposes a Releasable, + making it possible to acquire all permits and only release them all + when needed by closing the Releasable. The existing blockOperations() + method has been modified to delegate permit acquisition/releasing to this new + method. + + Relates to #33888 + +commit a90ef6bd6e3cac48ee289873ad3b6dcac83b62df +Author: Przemyslaw Gomulka +Date: Thu Nov 8 08:19:27 2018 +0100 + + HLRC: reindex API with wait_for_completion false (#35202) + + Extend High Level Rest Client Reindex API to support requests with + wait_for_completion=false. This method will return a TaskSubmissionResult with task identifier as string and results can be queried with Task API + + refers: #27205 + +commit a6073f513003ee812a63f50d1315b0f4e97bd1c1 +Author: Jason Tedor +Date: Wed Nov 7 22:25:37 2018 -0500 + + Add docs on JNA temp directory not being noexec (#35355) + + If the underlying mount point for the JNA temporary directory is mounted + noexec on Linux, then the JVM will not be able to map the native code in + as executable. This will prevent JNA from executing and will prevent + Elasticsearch from being able to execute some functions that rely on + native code (e.g., memory locking, and installing system call + filters). We do not want to get into the business of catching exceptions + and parsing messages towards this because these exception messages can + change on us. We also do not want to jump through a lot of hoops to + check the underlying mount point for noexec. Instead, we will rely on + documentation to address this problem. This commit adds to the important + system configuration section of the docs that the JNA temporary + directory is not on a mount point with the noexec mount option. + +commit 2f2090f5621d11e1bfefeb8b390a7bb5ad53a177 +Author: Martijn van Groningen +Date: Thu Nov 8 03:59:58 2018 +0100 + + [CCR] Adjust list of dynamic index settings that should be replicated (#35195) + + Adjust list of dynamic index settings that should be replicated + and added a test that verifies whether builtin dynamic index settings + are classified as replicated or non replicated (whitelisted). + +commit 4f4fc3b8f85d2af05e3106b4a9d2115aa00f3199 +Author: Jason Tedor +Date: Wed Nov 7 21:20:51 2018 -0500 + + Replicate index settings to followers (#35089) + + This commit uses the index settings version so that a follower can + replicate index settings changes as needed from the leader. + + Co-authored-by: Martijn van Groningen + +commit 7d052578969cb3f16200cbec9d8837b795bfaaff +Author: Tim Vernum +Date: Thu Nov 8 12:57:42 2018 +1100 + + Rename RealmConfig.globalSettings() to settings() (#35330) + + There is no longer a concept of non-global "realm settings". All realm + settings should be loaded from the node's settings using standard + Setting classes. + + This change renames the "globalSettings" field and method to simply be + "settings". + +commit 566979cc3159cec00b9b672eec59e45008dcf18f +Author: Tim Vernum +Date: Thu Nov 8 12:57:22 2018 +1100 + + [TEST] Cleanup FileUserPasswdStoreTests (#35329) + + The file realm has not supported custom filenames/locations since at + least 5.0, but this test still tried to configure them. + + Remove all configuration of file locations, and cleaned up a few other + warnings and deprecations + +commit a4d979cfc8f8cadf6552890184653c5be7651775 +Author: Ryan Ernst +Date: Wed Nov 7 15:09:09 2018 -0800 + + Scripting: Add back lookup vars in score script (#34833) + + The lookup vars under params (namely _fields and _source) were + inadvertently removed when scoring scripts were converted to using + script contexts. This commit adds them back, along with deprecation + warnings for those that should not be used. + +commit d7a4cef483f0fe3b6da62845d30346d0319cc258 +Author: Jake Landis +Date: Wed Nov 7 15:08:03 2018 -0600 + + watcher: Fix integration tests to ensure correct start/stop of Watcher (#35271) + + Ensure that Watcher is correctly started and stopped between tests for + SmokeTestWatcherWithSecurityIT, + SmokeTestWatcherWithSecurityClientYamlTestSuiteIT, + SmokeTestWatcherTestSuiteIT, WatcherRestIT, + XDocsClientYamlTestSuiteIT, and XPackRestIT + + The change here is to throw an `AssertionError` instead of `break;` to + allow the `assertBusy()` to continue to busy wait until the desired + state is reached. + + closes #33291, closes #29877, closes #34462, closes #30705, closes #34448 + +commit b1655c468ab0a58cabcda7c89e064f452e5ed8c6 +Author: Lee Hinman +Date: Wed Nov 7 13:40:19 2018 -0700 + + Remove ALL shard check in CheckShrinkReadyStep (#35346) + + Since it's still possible to shrink an index when replicas are unassigned, we + should not check that all copies are available when performing the shrink, since + we set the allocation requirement for a single node. + + Resolves #35321 + +commit ed8732b1619ed9ce01def1b8b8d87697e0aa484d +Author: Nhat Nguyen +Date: Wed Nov 7 15:26:30 2018 -0500 + + Use soft-deleted docs to resolve strategy for engine operation (#35230) + + A CCR test failure shows that the approach in #34474 is flawed. + Restoring the LocalCheckpointTracker from an index commit can cause both + FollowingEngine and InternalEngine to incorrectly ignore some deletes. + + Here is a small scenario illustrating the problem: + + 1. Delete doc with seq=1 => engine will add a delete tombstone to Lucene + + 2. Flush a commit consisting of only the delete tombstone + + 3. Index doc with seq=0 => engine will add that doc to Lucene but soft-deleted + + 4. Restart an engine with the commit (step 2); the engine will fill its + LocalCheckpointTracker with the delete tombstone in the commit + + 5. Replay the local translog in reverse order: index#0 then delete#1 + + 6. When process index#0, an engine will add it into Lucene as a live doc + and advance the local checkpoint to 1 (seq#1 was restored from the + commit - step 4). + + 7. When process delete#1, an engine will skip it because seq_no=1 is + less than or equal to the local checkpoint. + + We should have zero document after recovering from translog, but here we + have one. + + Since all operations after the local checkpoint of the safe commit are + retained, we should find them if the look-up considers also soft-deleted + documents. This PR fills the disparity between the version map and the + local checkpoint tracker by taking soft-deleted documents into account + while resolving strategy for engine operations. + + Relates #34474 + Relates #33656 + +commit c51dcb1f5d7e6269f9b72256bb40283f4489c5c0 +Author: Lee Hinman +Date: Wed Nov 7 13:24:21 2018 -0700 + + [ILM] Check shard and relocation status in AllocationRoutedStep (#35316) + + * [ILM] Check shard and relocation status in AllocationRoutedStep + + This is a follow-up from #35161 where we now check for started and relocating + state in `AllocationRoutedStep`. + + Resolves #35258 + +commit 8de3c6e618505e46c10f95e4360ec23c746d2ddf +Author: Martijn van Groningen +Date: Wed Nov 7 20:41:27 2018 +0100 + + Ignore date ranges containing 'now' when pre-processing a percolator query (#35160) + + Today when a percolator query contains a date range then the query + analyzer extracts that range, so that at search time the `percolate` query + can exclude percolator queries efficiently that are never going to match. + + The problem is that if 'now' is used it is evaluated at index time. + So the idea is to rewrite date ranges with 'now' to a match all query, + so that the query analyzer can't extract it and the `percolate` query + is then able to evaluate 'now' at query time. + +commit 0cc0fd2d1555c6954aa65aee875ff2db280bdb4f +Author: Simon Willnauer +Date: Wed Nov 7 20:23:35 2018 +0100 + + Add a frozen engine implementation (#34357) + + This change adds a `frozen` engine that allows lazily open a directory reader + on a read-only shard. The engine wraps general purpose searchers in a LazyDirectoryReader + that also allows to release and reset the underlying index readers after any and before + secondary search phases. + + Relates to #34352 + +commit f789d49fb35f88fae7d2a966593fa1a34cdbaa54 +Author: Vladimir Dolzhenko +Date: Wed Nov 7 20:18:11 2018 +0100 + + Put a fake allocation id on allocate stale primary command (#34140) + + removes fake allocation id after recovery is done + + Relates to #33432 + +commit 314b9ca44ca89443e2755077fc454737b6f54f63 +Author: Martijn van Groningen +Date: Wed Nov 7 20:16:26 2018 +0100 + + [CCR] Enforce auto follow pattern name restrictions (#35197) + + An auto follow pattern: + * cannot start with `_` + * cannot contain a `,` + * can be encoded in UTF-8 + * the length of UTF-8 encoded bytes is no longer than 255 bytes + +commit 4167c14d6ca2044d1de7360045bb663a1fb9150d +Author: Tal Levy +Date: Wed Nov 7 10:50:55 2018 -0800 + + [ILM] rolling upgrade tests (#35328) + + Adds basic rolling upgrade tests to check that lifecycles are still recognizable between rolling cluster upgrades and managed indices stay managed. + + This is a placeholder for discussing types of checks so they are ready once we backported + + This is a re-boot of the previous PR against index-lifecycle that needed to be + reverted due to CI bwc issues. #32828 + +commit 2117f4f3584f5f41f413ecd959ded550f14f891d +Author: Benjamin Trent +Date: Wed Nov 7 12:48:15 2018 -0600 + + [ML] Add Missing data checking class (#35310) + + * ML: Adding missing data check class + + * reverting bad change + + * Adding bucket + missing data object for returns + + * reverting unnecessary change + + * adding license header + + * Make client calls synchronous, akin to DatafeedJob + + * Fixing line length + + * Renaming things, addressing PR comments + +commit 2131e119d7a29e099e0307125d58ba960744f072 +Author: Simon Willnauer +Date: Wed Nov 7 18:43:27 2018 +0100 + + Apply `ignore_throttled` also to concrete indices (#35335) + + Today we only apply `ingore_throttled` to expansions from wildcards, + date math expressions and aliases. Yet, this is tricky since we might + have resolved certain expressions in pre-filter steps like security. + It's more consistent to apply this logic to all expressions including + concrete indices. + + Relates to #34354 + +commit 5ae03195d3195769e26767dba91f6965404d2fca +Author: Alpar Torok +Date: Wed Nov 7 18:36:02 2018 +0200 + + Make version field names more meaningful (#35334) + + * Consolidate the name of the qualified build version + + * Field name in response should not be redundant + +commit 021f80517fa326334424dd7e733644789d3a34fe +Author: Martijn van Groningen +Date: Wed Nov 7 17:08:35 2018 +0100 + + [CCR] Added HLRC support for pause follow API (#35216) + + * Moved `AcknowledgedResponse` to core package + * Made AcknowledgedResponse not abstract and provided a default parser, + so that in cases when the field name is not overwritten then there + is no need for a subclass. + + Relates to #33824 + +commit e572a21c4bb1d10d8606f356f8b0a829c006acf1 +Author: Chris Cho +Date: Thu Nov 8 01:01:35 2018 +0900 + + [Docs] Improve Convert Processor description (#35280) + + Sometimes users are confused about whether they can use the Convert Processor + for changing an existing fields type to other types even if the existing one is already + ingested. This confusion is from the first line of description. Changing this and also + adding a some detail to the code snippet. + +commit efe9beca7bbb706888b01e0ad82b488ede9628ff +Author: Jack Conradson +Date: Wed Nov 7 07:52:12 2018 -0800 + + [Painless] Removes extraneous compile method (#35323) + + With the removal of SearchScript (#34730), an extraneous compile method + was left behind in PainlessScriptEngine. This change removes the method and + updates the tests that depend on it to use the main compile method which + gives better test coverage. + +commit e685cfe8f985e1669b0594bbd5b2300c35d8bbb9 +Author: Martijn van Groningen +Date: Wed Nov 7 13:23:30 2018 +0100 + + [CCR] Fail with a better error if leader index is red (#35298) + + as part of fetching history uuids from leader index. + +commit 9a319ec412074b1a39543f0e7f2c2afcbfdcbf02 +Author: Alpar Torok +Date: Wed Nov 7 14:22:00 2018 +0200 + + Remove ILM specific gradle extra args (#35296) + +commit 8a85b2eadadc2307d30afd24e24c25b6dfb8d298 +Author: Alpar Torok +Date: Wed Nov 7 14:01:05 2018 +0200 + + Remove build qualifier from server's Version (#35172) + + With this change, `Version` no longer carries information about the qualifier, + we still need a way to show the "display version" that does have both + qualifier and snapshot. This is now stored by the build and red from `META-INF`. + +commit a467a816bc1469837edbdb237e1bc99e4650ca2d +Author: Vladimir Dolzhenko +Date: Wed Nov 7 12:52:11 2018 +0100 + + Fix DeleteRequest / GetRequest / UpdateRequest / ExplainRequest validation for null and/or empty id/type (#35314) + + + Closes #35297 + +commit 42dcdd0aa8b887097c1e1626da3202492a6e350c +Author: Marios Trivyzas +Date: Wed Nov 7 12:08:32 2018 +0100 + + SQL: Improve CircuitBreaker logic for SqlParser (#35300) + + Grammar's identifiers can be completely skipped from counting depths + as they just add another level to the tree and they are always children + of some other expression which gets counted. + + Increased maximum depth from 100 to 200. After testing on production + configuration with -Xss1m, depths of at least 250 can be used, so being + conservative we put the limit lower. + + Fixes: #35299 + +commit 9f3effd690ac5e16d7d0559849586fff7e5068fd +Author: Alexander Reelsen +Date: Wed Nov 7 09:03:35 2018 +0100 + + Watcher: Ignore system locale/timezone in croneval CLI tool (#33215) + + The elasticsearch-croneval CLI tool uses local dates to display when + something gets triggered the next time. This is very confusing. + + This commit ensures, that UTC and local timezone times will be written + out. + + The output looks like this and contains localized dates for each trigger + date as well as for `now`. + + Now is [Tue, 28 Aug 2018 17:23:51 +0000] in UTC, local time is [ᏔᎵᏁ, 28 ᎦᎶ 2018 12:23:51 -0500] + Here are the next 10 times this cron expression will trigger: + 1. Mon, 2 Jan 2040 11:00:00 +0000 + ᏉᏅᎯ, 2 ᎤᏃ 2040 06:00:00 -0500 + 2. ... + + This also removes an old outstanding TODO to use the jopt parsing to + cast the count to an integer instead of doing it ourselves. + +commit b4173c83934a4858d013463cd7d9e024faf5b7ef +Author: Tim Vernum +Date: Wed Nov 7 18:21:30 2018 +1100 + + Remove deprecated RealmConfig constructor (#35327) + + This removes an obsolete constructor that was still being called from + some tests. + + Relates: #30241 + +commit 2395e16d849fce60c9b7a63c92b5882080e6fde0 +Author: Martijn van Groningen +Date: Wed Nov 7 07:38:44 2018 +0100 + + [CCR] Change resume follow api to be a master node action (#35249) + + In order to start shard follow tasks, the resume follow api already + needs execute N requests to the elected master node. + + The pause follow API is also a master node action, which would make + how both APIs execute more consistent. + +commit 1614107d59ba28748d207e68a4e3ccac9d0d5706 +Author: CJ Cenizal +Date: Tue Nov 6 17:22:50 2018 -0800 + + Docs: Update CONTRIBUTING.md with shortcut command for assembling only the tar distribution (#35276) + + * Add --parallel flag. + +commit f395b1eace8297aeef16648473a7e7f3500b671e +Author: Tim Brooks +Date: Tue Nov 6 17:58:20 2018 -0700 + + Open node connections asynchronously (#35144) + + This is related to #29023. Additionally at other points we have + discussed a preference for removing the need to unnecessarily block + threads for opening new node connections. This commit lays the groudwork + for this by opening connections asynchronously at the transport level. + We still block, however, this work will make it possible to eventually + remove all blocking on new connections out of the TransportService + and Transport. + +commit ec53288fc0c94c4f514b24b2230671c7ec0316ed +Author: Julie Tibshirani +Date: Tue Nov 6 14:33:48 2018 -0800 + + Remove include_type_name from the relevant APIs. (#35192) + + We've decided that the bulk, delete, get, index, update, and search APIs should not + contain this request parameter, and we will instead accept both typed and typeless calls. + +commit 6f6b26516632c97477d150b397596739b500848a +Author: Jay Modi +Date: Tue Nov 6 13:12:24 2018 -0700 + + HLRC: add support for the clear realm cache API (#35163) + + This change adds support for clearing the cache of a realm. The realms + cache may contain a stale set of credentials or incorrect role + assignment, which can be corrected by clearing the cache of the entire + realm or just that of a specific user. + + Relates #29827 + +commit a4b26febcb1cb756640a88aaaee5ded3bf4e987e +Author: Yannick Welsch +Date: Tue Nov 6 19:33:48 2018 +0100 + + Register Azure max_retries setting (#35286) + + This commit properly registers the Azure max_retries setting in the settings infrastructure, + allowing this setting to be actually used. + +commit 3c18aa0c15fcfe2222720194a41ad2c6e7365ab1 +Author: javanna +Date: Tue Nov 6 17:28:39 2018 +0100 + + add version 6.4.4 + +commit 7e356ac29ba3cf676416b0b1c6c716cfeacedc43 +Author: David Turner +Date: Tue Nov 6 18:10:29 2018 +0000 + + [Zen2] Introduce auto_shrink_voting_configuration setting (#35217) + + Today we allow the user to set the minimum size of a voting configuration. On + reflection we would rather this was simply '3' where possible, and we can use + the retirement API to control the removal of nodes more explicitly. + + This change replaces the old reconfigurator setting with a new one, + `cluster.auto_shrink_voting_configuration`, which determines whether + Elasticsearch should automatically remove nodes from the voting configuration + or not. + +commit 065264d57c04397a0d7009c67ad709be19693535 +Author: Zachary Tong +Date: Tue Nov 6 12:18:29 2018 -0500 + + [Docs] Add painless context details for bucket_script (#35142) + + Fleshes out the details of the bucket_script context, adds an example + and corresponding doc test + +commit 432c51826628486e0f692a44bf36a13e5bf47c64 +Author: Andrei Stefan +Date: Tue Nov 6 18:42:57 2018 +0200 + + Upgrade jline to 3.8.2 (#35288) + +commit c3e4575f61a20528f605458ff7e779a5f08936c1 +Author: Andrei Stefan +Date: Tue Nov 6 17:51:52 2018 +0200 + + SQL: new SQL CLI logo (#35261) + + Added new SQL CLI logo representing the Elastic logo "painted" with + "Elastic" words, "SQL" under the logo and version on the last line + +commit 348c28d1d1b38e49900caf9937221313cb87d57b +Author: Nik Everett +Date: Tue Nov 6 10:49:54 2018 -0500 + + Logger: Merge ESLoggerFactory into Loggers (#35146) + + `ESLoggerFactory` is now not particularly interesting and simple enought + to fold entirely into `Loggers. So let's do that. + + Closes #32174 + +commit 3ee37b425b481139c433a387e5178a63f19d495b +Author: Gytis Šk +Date: Tue Nov 6 17:45:10 2018 +0200 + + Docs: Add section about range query for range type (#35222) + + This makes their interaction more discoverable. + +commit a85b4f42ca3965e1c94f621948b3b48fa66611fb +Author: Tal Levy +Date: Tue Nov 6 07:46:25 2018 -0800 + + [ILM] change remove-policy-from-index http method from DELETE to POST (#35268) + + The remove-ilm-from-index API was using the DELETE http method + to signify that something is being removed. Although, metadata + about ILM for the index is being deleted, no entity/resource + is being deleted during this operation. POST is more in line with + what this API is actually doing, it is modifying the metadata for + an index. As part of this change, `remove` is also appended to the path + to be more explicit about its actions. + +commit a937d7f5f3bc92fcbc2e789ec97b16f9e652194a +Author: Martijn van Groningen +Date: Tue Nov 6 16:04:37 2018 +0100 + + [CCR] Forgot missing return statement, + + Error was thrown if leader index had no soft deletes enabled, but it then continued creating the follower index. + + The test caught this bug, but very rarely due to timing issue. + + Build failure instance: + + ``` + 1> [2018-11-05T20:29:38,597][INFO ][o.e.x.c.LocalIndexFollowingIT] [testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes] before test + 1> [2018-11-05T20:29:38,599][INFO ][o.e.c.s.ClusterSettings ] [node_s_0] updating [cluster.remote.local.seeds] from [[]] to [["127.0.0.1:9300"]] + 1> [2018-11-05T20:29:38,599][INFO ][o.e.c.s.ClusterSettings ] [node_s_0] updating [cluster.remote.local.seeds] from [[]] to [["127.0.0.1:9300"]] + 1> [2018-11-05T20:29:38,609][INFO ][o.e.c.m.MetaDataCreateIndexService] [node_s_0] [leader-index] creating index, cause [api], templates [random-soft-deletes-templat + e, one_shard_index_template], shards [2]/[0], mappings [] + 1> [2018-11-05T20:29:38,628][INFO ][o.e.c.r.a.AllocationService] [node_s_0] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[leader- + index][0]] ...]). + 1> [2018-11-05T20:29:38,660][INFO ][o.e.x.c.a.TransportPutFollowAction] [node_s_0] [follower-index] creating index, cause [ccr_create_and_follow], shards [2]/[0] + 1> [2018-11-05T20:29:38,675][INFO ][o.e.c.s.ClusterSettings ] [node_s_0] updating [cluster.remote.local.seeds] from [["127.0.0.1:9300"]] to [[]] + 1> [2018-11-05T20:29:38,676][INFO ][o.e.c.s.ClusterSettings ] [node_s_0] updating [cluster.remote.local.seeds] from [["127.0.0.1:9300"]] to [[]] + 1> [2018-11-05T20:29:38,678][INFO ][o.e.x.c.LocalIndexFollowingIT] [testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes] after test + 1> [2018-11-05T20:29:38,678][INFO ][o.e.x.c.LocalIndexFollowingIT] [testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes] [LocalIndexFollowingIT#testDoNotCreateFoll + owerIfLeaderDoesNotHaveSoftDeletes]: cleaning up after test + 1> [2018-11-05T20:29:38,678][INFO ][o.e.c.m.MetaDataDeleteIndexService] [node_s_0] [follower-index/TlWlXp0JSVasju2Kr_hksQ] deleting index + 1> [2018-11-05T20:29:38,678][INFO ][o.e.c.m.MetaDataDeleteIndexService] [node_s_0] [leader-index/FQ6EwIWcRAKD8qvOg2eS8g] deleting index + FAILURE 0.23s J0 | LocalIndexFollowingIT.testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes <<< FAILURES! + > Throwable #1: java.lang.AssertionError: + > Expected: + > but: was + > at __randomizedtesting.SeedInfo.seed([7A3C89DA3BCA17DD:65C26CBF6FEF0B39]:0) + > at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20) + > at org.elasticsearch.xpack.ccr.LocalIndexFollowingIT.testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes(LocalIndexFollowingIT.java:83) + > at java.lang.Thread.run(Thread.java:748) + ``` + + Build failure: https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+6.5+intake/46/console + +commit 1be64a7b8b047773f149124524a1bb5f490a9271 +Author: Marios Trivyzas +Date: Tue Nov 6 15:40:16 2018 +0100 + + SQL: Fix null handling for AND and OR in SELECT (#35277) + + Override `process()` in `BinaryLogicProcessor` which doesn't immediately + return null if left or right argument is null, which is the behaviour of + `process()` of the parent class `BinaryProcessor`. + + Also, add more tests for `AND` and `OR` in SELECT clause with literal. + + Fixes: #35240 + +commit 2bf843e768f65d2531ad97fcb8c0b91779182913 +Author: Tal Levy +Date: Tue Nov 6 06:09:49 2018 -0800 + + [TEST] Mute ChangePolicyForIndexIT#testChangePolicyForIndex + +commit 6e58284683794f22d90591923df1af006259f735 +Author: Simon Willnauer +Date: Tue Nov 6 13:49:42 2018 +0100 + + Serialize ignore_throttled also to 6.6 after backport + +commit 366f79cef219ac04cd7080dd7153305848dadb8e +Author: Alpar Torok +Date: Tue Nov 6 14:43:48 2018 +0200 + + Check for java 11 in buildSrc (#35260) + +commit 17b7d2efcb5f94596f2bc9bd6d8710db7beceab1 +Author: javanna +Date: Tue Nov 6 13:36:22 2018 +0100 + + [TEST] increase await timeout in RemoteClusterConnectionTests + + We have seen an improvement when we bumped the timeout from 1s to 5s, but there are still a few failures for this tests. With this commit we bump the timeout to 10 seconds hoping it will stop all the failures. + +commit 58f3e815556e15a7916a89bd5fae60d39e90b7a8 +Author: Alpar Torok +Date: Tue Nov 6 14:00:40 2018 +0200 + + Add missing up-to-date configuration (#35255) + + * Add missing up-to-date configuration + + The source properties file and the dynamic elasticsearch version + (set based on properties ) were missing from task outputs leading + to the task being incorrectly considered up to date. + + Closes #35204 + +commit 999f8f5850bed0f0c0b3b2730da57a68947d1c82 +Author: Jim Ferenczi +Date: Tue Nov 6 12:15:16 2018 +0100 + + Adapt Lucene BWC version + + Bump the Lucene version used by ES 6.6 now that the 6.x branch is upgraded to Lucene 7.6. + +commit 75e9a639eeb0d6142cd17e4edf34de22da57aebf +Author: Costin Leau +Date: Tue Nov 6 13:12:24 2018 +0200 + + SQL: Introduce Coalesce function (#35253) + + Add Coalesce conditional for replacing null values + + Fix #35060 + +commit a5e1f4d3a2b6f03f5b6268555ebb4271ea3c153b +Author: Nick Knize +Date: Tue Nov 6 04:55:23 2018 -0600 + + Upgrade to lucene-8.0.0-snapshot-31d7dfe6b1 (#35224) + +commit 9f4b93fd5e9119f57463f4d9e8a2520ecb5c35ce +Author: Alan Woodward +Date: Tue Nov 6 09:02:40 2018 +0000 + + Fix failing ICU tests (#35207) + + Fixes #35173 + +commit 833e0f8ecf43be368da2490d71aaa2c88785b5d7 +Author: Simon Willnauer +Date: Tue Nov 6 09:45:30 2018 +0100 + + Prevent throttled indices to be searched through wildcards by default (#34354) + + Today if a wildcard, date-math expression or alias expands/resolves + to an index that is search-throttled we still search it. This is likely + not the desired behavior since it can unexpectedly slow down searches + significantly. + + This change adds a new indices option that allows `search`, `count` + and `msearch` to ignore throttled indices by default. Users can + force expansion to throttled indices by using `ignore_throttled=true` + on the rest request to expand also to throttled indices. + + Relates to #34352 + +commit 46c238d7920b6cf3445c5e69e2b4476b399ff769 +Author: Martijn van Groningen +Date: Tue Nov 6 08:42:47 2018 +0100 + + [CCR] Improve error when operations are missing (#35179) + + Improve error when operations are missing + +commit cac67f8bccc8599cd2b9cc96ca78899158aa644e +Author: Martijn van Groningen +Date: Tue Nov 6 08:00:34 2018 +0100 + + [CCR] Add extra validation to unfollow api (#35245) + + Validate whether the follow index actually exists and + whether the follow index actually has custom ccr metadata. + +commit 14c8a483d5cf33f56d20acef0d5dbb55dfad06db +Author: Jeff Soloshy +Date: Mon Nov 5 22:49:18 2018 -0800 + + [Docs] Minor formatting and wording fixes (#35278) + +commit 7cd27d1a96b92a78800b4afacedc5c625ee04777 +Author: Tim Vernum +Date: Tue Nov 6 16:59:11 2018 +1100 + + [DOCS] Fix missing float in breaking changes + +commit 3776de5f202a9248e5a07b20bcd11d3d9bbaf269 +Author: Tim Vernum +Date: Tue Nov 6 15:26:12 2018 +1100 + + HLRC: Add InvalidateToken security API (#35114) + + This change adds the Invalidate Token API + (DELETE /_xpack/security/oauth2/token) to the Elasticsearch + High Level Rest Client. + + Relates: #29827 + +commit 574ec6686ed5edfbc9876356d8e8c53aff107289 +Author: Tim Vernum +Date: Tue Nov 6 14:56:50 2018 +1100 + + Include realm type in Security Realm setting keys (#30241) + + This moves all Realm settings to an Affix definition. + However, because different realm types define different settings + (potentially conflicting settings) this requires that the realm type + become part of the setting key. + + Thus, we now need to define realm settings as: + + xpack.security.authc.realms: + file.file1: + order: 0 + + native.native1: + order: 1 + + - This is a breaking change to realm config + - This is also a breaking change to custom security realms (SecurityExtension) + +commit 00e66bab362b1a8cd72c5fc483b8523c3888890e +Author: Shaunak Kashyap +Date: Mon Nov 5 16:27:05 2018 -0800 + + [Elasticsearch Monitoring] Collect only display_name (for now) (#35265) + + This is a forward port of some of the changes made in #8445, specifically the change mentioned in https://github.com/elastic/elasticsearch/pull/34023#issuecomment-433212636. + + Currently, in master, the `cluster_stats` collector collects _all_ cluster metadata and indexes it into `.monitoring-es-*`. However, per the discussion linked to above, we decided to collect _only_ the `display_name` cluster metadata setting for now. This PR makes this change. + +commit 8280596f1665c269beadf51765f40e4c1e559bfe +Author: lcawl +Date: Mon Nov 5 15:30:35 2018 -0800 + + [DOCS] Fixes formatting error in auditing settings + +commit 9150a93b1cf0202b288b36863614947d581a6e04 +Author: Jake Landis +Date: Mon Nov 5 17:16:42 2018 -0600 + + ingest: dot_expander_processor prevent null add/append to source document (#35106) + + * don't allow null values to be added or appended to the + source document if the field does not exist. + +commit 2fb3d1a46521de25ca1a726dc2b6cb82647fda2d +Author: David Turner +Date: Mon Nov 5 21:54:53 2018 +0000 + + [Zen2] Fix some rarely-failing tests (#35198) + + Recent changes have left a few Zen2 tests occasionally failing. This commit + fixes them. + +commit 9ac7af6b3e3bf21fdc618531b6ac1f332651cfbe +Author: Marios Trivyzas +Date: Mon Nov 5 22:23:07 2018 +0100 + + SQL: Introduce NotEquals node to simplify expressions (#35234) + + Add NotEquals node in parser to simplify expressions so that != is + no longer translated internally to NOT( = ) + + Closes: #35210 + Fixes: #35233 + +commit f72ef9b5fdb0699c320e5c983c0fe6122aaeea94 +Author: Nik Everett +Date: Mon Nov 5 16:16:00 2018 -0500 + + Build: Pull "skip assemble on qa" to common build (#35214) + + Pull all of the logic that we use to skip the `assemble` and + `dependenciesInfo` tasks on `qa` projects into one spot in our root + build file. + +commit 216c761a5dd81e65ca9826fb0793ddc53cd2ff27 +Author: Armin Braun +Date: Mon Nov 5 20:40:27 2018 +0100 + + MINOR: Remove Dead Code in Routing (#35074) + + * MINOR: Remove Dead Code in Routing + +commit 3ee004c7b9a769ba34f11dd6ebd1dbd1dd336f7f +Author: Lee Hinman +Date: Mon Nov 5 12:02:59 2018 -0700 + + Add dedicated step for checking shrink allocation status (#35161) + + This adds a new step for checking whether an index is allocated correctly based + on the rules added prior to running the shrink step. It also fixes a bug where + for shrink we are not allowed to have the shards relocating for the shrink step. + + This also allows us to simplify AllocationRoutedStep and provide better + feedback in the step info for why either the allocation or the shrink checks + have failed. + + Resolves #34938 + +commit 842809ef37f50664c76ca15109724d63b6a22ed7 +Author: Alexander Zhukov +Date: Mon Nov 5 20:31:35 2018 +0200 + + Doc: Drop an extra 'a' in snapshots.asciidoc (#35251) + +commit 59a43180a5cbdc954eadb42f45696c2082f45284 +Author: Nik Everett +Date: Mon Nov 5 12:28:01 2018 -0500 + + Reindex: Clean up docs around multi-index + + We have an example in `reindex`'s docs about copying from many indices + at once. It doesn't work at the moment because we only allow a single + type per index. We didn't notice it in the docs tests because those + tests didn't copy any documents. This change: + 1. Adds documents to the docs tests to fully exercise the snippet. + 2. Fixes the example by moving all copied documents to the same type. + 3. Moves the note about id collisions and expands on it because it is + even more likely than before. + + Closes #35150 + +commit fd82813660ea726815f6020ae4a1f3a468f2b195 +Author: Marios Trivyzas +Date: Mon Nov 5 18:45:58 2018 +0100 + + SQL: [tests] Fix select csv spec and enable it (#35239) + + Also, replace `||` and `&&` with `OR` and `AND` as the former are not + SQL standard operators. + +commit c2766b65cfa3648838d670f601f20f14886a44da +Author: Jake Landis +Date: Mon Nov 5 11:44:10 2018 -0600 + + ingest: raise visibility of ingest plugin documentation (#35048) + + * move the set security user processor to the main documentation + * link to plugin processors + + part of #33188 + +commit a95f05780d69572270ef7c2b25bd38b10a73b6d9 +Author: Armin Braun +Date: Mon Nov 5 17:53:02 2018 +0100 + + NETWORK: Remove Dead Code from Netty4CorsConfig (#34947) + + * Same as #34324 for the Netty transport, the `isNullOriginAllowed` setting is always false + +commit 0fbb8a16bcca543e0507a558a27bdecab7270ee6 +Author: Gordon Brown +Date: Mon Nov 5 09:20:43 2018 -0700 + + Skip Rollover step if next index already exists (#35168) + + If the Rollover step would fail due to the next index in sequence + already existing, just skip to the next step instead of going to the + Error step. + + This prevents spurious `ResourceAlreadyExistsException`s created by + simultaneous RolloverStep executions from causing ILM to error out + unnecessarily. + +commit 4f35eea8fe614ed686a6617e9d11ac695b1189a6 +Author: Yannick Welsch +Date: Mon Nov 5 16:15:19 2018 +0100 + + [TEST] Fix testConcurrentTermIncreaseOnReplicaShard + + This test has a bug that got introduced during the refactoring of #32442. With 2 concurrent term increments, + we can only assert under the operation permit that we are in the correct operation term, not that there is + not already another term bump pending. + + Closes #34862 + +commit 3473217563c6a05d1e1491931c1271e75c2d3e57 +Author: Lee Hinman +Date: Mon Nov 5 08:17:15 2018 -0700 + + Remove Joda usage from ILM (#35220) + + This commit removes the Joda time usage from ILM and the HLRC components of ILM. + It also fixes an issue where using the `?human=true` flag could have caused the + parser not to work. These millisecond fields now follow the standard we use + elsewhere in the code, with additional fields added iff the `human` flag is + specified. + + This is a breaking change for ILM, but since ILM has not yet been released, no + compatibility shim is needed. + +commit a4442dacd7e215cdedc4419028c1bb2345bc166e +Author: Benjamin Trent +Date: Mon Nov 5 08:56:53 2018 -0600 + + HLRC: Add ML API PUT filter (#35175) + +commit 9d28a104e132f4f8f5af9519336522ead8d00866 +Author: Vladimir Dolzhenko +Date: Mon Nov 5 15:02:09 2018 +0100 + + Discovery EC2: Utilize Amazon SDK to resolve EC2 metadata server (#35246) + + * Discovery EC2: Utilize Amazon SDK to resolve EC2 metadata server + + Closes #35141 + +commit 02043a2260d225175dea730d32a29e40d1cc5f67 +Author: Christoph Büscher +Date: Mon Nov 5 13:31:13 2018 +0100 + + [Tests] Fix rare edge case in SimpleQueryStringBuilderTests (#35201) + + If the random query string is "now" by accident _and_ we are also not setting + some field names to use explicitely, then we can hit the "mapped_date" field + from default test setup. This correctly leads to the query being was marked as + not cacheable, but we assume and check so later. This change fixes this rare + edge case by making sure we don't hit the "date" field in this rare cases. + + Closes #35183 + +commit 409050e8de4cacbfac4b68ab12bbb928437a4499 +Author: Alexander Reelsen +Date: Mon Nov 5 13:08:18 2018 +0100 + + Refactor: Remove settings from transport action CTOR (#35208) + + As settings are not used in the transport action constructor, this + removes the passing of the settings in all the transport actions. + +commit ddda2d419cb49c8df0df70b8ce8d4c8b7322ef02 +Author: Martijn van Groningen +Date: Mon Nov 5 12:51:42 2018 +0100 + + [CCR] Change max_read_request_size default (#35247) + + This changes the max_read_request_size default from unlimited to 32MB. + +commit 85f8458f0641554cdc4f85bbba4f465f70529031 +Author: David Kyle +Date: Mon Nov 5 11:21:28 2018 +0000 + + [ML] Add comment describing test behaviour + +commit a2c22ad7aeb0f8d5148677dd7222ad43afee9a17 +Author: Mayya Sharipova +Date: Sun Nov 4 18:29:07 2018 -0500 + + Small corrections to HLRC doc for _termvectors (#35221) + + Relates to #33447 + +commit 28078642b34e1e655c761ae1e0cf37b5449890b8 +Author: Boaz Leskes +Date: Sun Nov 4 13:43:33 2018 +0100 + + Engine.newChangesSnapshot may cause unneeded refreshes if called concurrently (#35169) + + When the engine is asked for historical operations, we check if some of the requested operations + are not yet refreshed and if so we refresh before returning the operations. The refresh check is + based on capturing the local checkpoint before each refresh and comparing that value to the one + requested when `newChangesSnapshot` was called. If the requested range is above the captured + local checkpoint we issue a refresh. + + This can currently cause unneeded extra refreshes if the method is called concurrently which may cause unwanted degradation in indexing performance. This is especially relevant for CCR where we always ask for a range below the global checkpoint. That range is guaranteed to be below the local + checkpoint of the shard and one refresh is enough to serve multiple changes requests. + + This commit fixes this by introducing a dedicated mutex to make sure the test for whether a refresh + is needed actually wait for concurrents for concurrent refreshes that were caused by another + change refresh. + + Note that this is not a big change in semantics as refreshes are serialized by lucene anyway. I also + opted not to keep the synchronization to the changes snapshot request only even if in theory we + can apply it to all refreshes, not matter where they come from. + +commit 54e1231ebd7b637dd8cd8beb3769b8825d80446a +Author: Nhat Nguyen +Date: Sat Nov 3 10:00:54 2018 -0400 + + CCR/TEST: Limit indexing docs in FollowerFailOverIT (#35228) + + The suite FollowerFailOverIT is failing because some documents are not + replicated to the follower. Maybe the FollowTask is not working as + expected or the background indexers eat all resources while the follower + cluster is trying to reform after a failover; then CI is not fast enough + to replicate all the indexed docs within 60 seconds (sometimes I see 80k + docs on the leader). + + This commit limits the number of documents to be indexed into the leader + index by the background threads so that we can eliminate the latter + case. This change also replaces a docCount assertion with a docIds + assertion so we can have more information if these tests fail again. + + Relates #33337 + +commit 855ab3fa1e534d6469d94b48d257cc724d0bc59b +Author: Nhat Nguyen +Date: Fri Nov 2 21:31:36 2018 -0400 + + Add equals/hashCode to SeqNoStats (#35223) + + This commit adds equals/hashCode to SeqNoStats so we can verify it wholly in tests. + +commit 44f08717bac83af48660e5d98928814b066788fa +Author: Jack Conradson +Date: Fri Nov 2 16:07:54 2018 -0700 + + [Scripting] Make Max Script Length Setting Dynamic (#35184) + + This changes the current script.max_size_in_bytes to be dynamic so it can be + set through the cluster settings API. This setting is also applied to inline scripts + in the compile method of ScriptService to prevent excessively long inline + scripts from being compiled. The script length limit is removed from Painless as + this is no longer necessary with the protection in compile. + +commit 0166388d7427643d565dc0e0af9a8cc58c804b24 +Author: Tim Brooks +Date: Fri Nov 2 16:31:19 2018 -0600 + + Use single netty event loop group for transports (#35181) + + Currently we create a new netty event loop group for client connections + and all server profiles. Each new group creates new threads for io + processing. This means 2 * num of processors new threads for each group. + A single group should be able to handle all io processing (for the + transports). This also brings the netty module inline with what we do + for nio. + + Additionally, this PR renames the worker threads to be the same for + netty and nio. + +commit 3c36ba1f5ece5b025bf027a9ee8912336126035c +Author: Lisa Cawley +Date: Fri Nov 2 14:23:13 2018 -0700 + + [DOCS] Move CCR overview pages (#34918) + +commit ec1c2c4a76c149c0457f53eb3e188980f62bb675 +Author: Jay Modi +Date: Fri Nov 2 14:30:43 2018 -0600 + + Remove AbstractComponent usage in x-pack core (#35187) + + This change removes the use of AbstractComponent in the security + module. The classes now declare their own loggers. + + Relates #34488 + +commit b3da3eae08a05f24b16c8fb5e6247c9e10439826 +Author: Gordon Brown +Date: Fri Nov 2 11:09:48 2018 -0600 + + [ILM] Fix race condition in test (#35143) + + Previously, testRunStateChangePolicyWithNextStep asserted that the + ClusterState before and after running the steps were equal. The test + only passed due to a race condition: The latch would be triggered by the + step execution, but the cluster state update thread would continue + running before committing the change to the cluster state. This allowed + the test to read the old cluster state and pass the equality check about + 99.99% of the time. + + The test now waits for the new cluster state to be committed before + checking that it is _not_ equal to the old cluster state. + +commit d6e44129b1cbe77562195f970716ec024f210865 +Author: Nhat Nguyen +Date: Fri Nov 2 12:27:48 2018 -0400 + + TEST: Only check max_seq_no_of_updates when rollback (#35170) + + Currently, we assume that rollback always happens in the test + testRestoreLocalHistoryFromTranslogOnPromotion. However, if the global + checkpoint equals max_seq_no, we won't rollback. This causes the + max_seq_no_of_updates assertion failed because max_seq_no_of_updates + won't be advanced to the global checkpoint. With this commit, we assert + max_seq_no_of_updates in two different paths. + +commit 9b3d581339d2956b7daefaeddff401898a6d1414 +Author: Gordon Brown +Date: Fri Nov 2 09:38:40 2018 -0600 + + [DOCS] Remove link to nonexistent ILM API + +commit 1e241190eb2596a258fcabe0b6408fdaac4c7603 +Author: Jason Tedor +Date: Fri Nov 2 11:11:43 2018 -0400 + + Disable assemble task from ILM qa projects + + This commit disables the assemble tasks from all ILM qa projects. These + projects do not have an assemble task to execute. + +commit 43cff56aecd2877a3be758b3280fd5071f477d02 +Author: David Kyle +Date: Fri Nov 2 15:12:47 2018 +0000 + + Mute FollowerFailOverIT.testFollowIndexAndCloseNode + + Issue #33337 + +commit e2af849f702973cbec4d96e1da5ecbbc78e4db5f +Author: Alan Woodward +Date: Fri Nov 2 15:12:09 2018 +0000 + + Move ObjectPath and XContentUtils to libs/x-content (#34803) + + These are generally useful utility classes that do not need to live in the Watcher code + +commit 6b08d5fc89e2e866b62c28dc1ee01364e0643e1a +Author: Jay Modi +Date: Fri Nov 2 08:33:26 2018 -0600 + + Remove use of AbstractComponent in security (#35186) + + This change removes the use of AbstractComponent in the security + module. The classes now declare their own loggers. + + Relates #34488 + +commit 34677b9c83f967011440c018b7676b3701c26649 +Author: Milan Mrdjen +Date: Fri Nov 2 14:21:19 2018 +0100 + + HLRC: Add document _count API (#34267) + + Add `count()` api method, `CountRequest` and `CountResponse` classes to HLRC. Code in server module is unchanged. + + Relates to #27205 + +commit 12072b200efedfbc894bd33d781a392b26789071 +Author: Vladimir Dolzhenko +Date: Fri Nov 2 13:52:15 2018 +0100 + + Run code snippets in ReindexDocumentationIT (#35165) + + + Closes #32093 + +commit e753e12f613d6abf5363d029b0dfda9aa5547721 +Author: Nhat Nguyen +Date: Fri Nov 2 08:49:55 2018 -0400 + + Do not alloc full buffer for small change requests (#35158) + + Today we always allocate a full buffer (1024 elements) in a + LuceneChangesSnapshot even though the requesting size is smaller. + With this change, we will use the requesting size as the buffer size if + it's smaller than the default batch size; otherwise uses the default + batch size. + +commit 7fb95be8bd6e736c279ced1e4b31791330d2c14c +Author: Benjamin Trent +Date: Fri Nov 2 07:46:53 2018 -0500 + + Moving xpack.plugin.core ml response to use new base class (#35157) + + * Moving xpack.plugin.core ml response to use new base class + + * changing back to streamable + +commit 8f592bda27f7d9805276f16493c931b553a50f6b +Author: Colin Goodheart-Smithe +Date: Fri Nov 2 12:27:51 2018 +0000 + + Removes link to deleted page + +commit ccbe80c3a00524f69ef06e4855686065733955fd +Author: Daniel Mitterdorfer +Date: Fri Nov 2 13:12:44 2018 +0100 + + Introduce durability of circuit breaking exception + + With this commit we differentiate between permanent circuit breaking + exceptions (which require intervention from an operator and should not + be automatically retried) and transient ones (which may heal themselves + eventually and should be retried). Furthermore, the parent circuit + breaker will categorize a circuit breaking exception as either transient + or permanent based on the categorization of memory usage of its child + circuit breakers. + + Closes #31986 + Relates #34460 + +commit 0aa66b077a53e102b4f251841ffb33b59a6a5a84 +Merge: fc6e1f7f3fb 4875d6fb0bf +Author: Colin Goodheart-Smithe +Date: Fri Nov 2 11:20:54 2018 +0000 + + Merge branch 'master' into index-lifecycle + +commit 4875d6fb0bfd5020e62996c1f761f9685104959d +Author: Nhat Nguyen +Date: Fri Nov 2 07:01:46 2018 -0400 + + CCR: Add NodeClosedException to retryable list (#35191) + + This change adds NodeClosedException to the retry-able exception list. + +commit fc6e1f7f3fb670d87646fd20b9c94248542d8ef6 +Merge: b260406ecf6 340363b9fa7 +Author: Colin Goodheart-Smithe +Date: Fri Nov 2 10:56:35 2018 +0000 + + Merge branch 'master' into index-lifecycle + +commit 340363b9fa7a3e710737af10efdbf9801eea2b57 +Author: Marios Trivyzas +Date: Fri Nov 2 10:37:54 2018 +0100 + + SQL: Fix null handling for IN => painless script (#35124) + + Include `null` literals when generating the painless script for `IN` expressions. + Previously, they were skipped, because of an issue that had been fixed with #35108. + + Fixes: #35122 + +commit b432cbdafee7efcc7120d36e4f6234e690ce0c20 +Author: Alpar Torok +Date: Fri Nov 2 09:46:05 2018 +0200 + + Fix upgrade tests involving snapshot + + Fixes + ``` + Could not determine the dependencies of task ':x-pack:qa:full-cluster-restart:with-system-key:v5.6.13#oldClusterTestCluster#node1.copyBwcPlugins'. + ``` + +commit 19d6cf1b9ed1938f8488caaa9071a9db52c0bf1e +Author: Martijn van Groningen +Date: Fri Nov 2 08:02:17 2018 +0100 + + [CCR] Change response classes to not use from Streamable. (#35085) + + Only the response classes of get auto follow pattern, the follow and stats APIs + were moved away from Streamable. The other APIs use `AcknowledgedResponse` + or `BaseTasksResponse` as response class and + moving that class away from Streamable is a bigger change. + +commit 598b0a0eb409e7a7f305c8fa4905b236c25f91e0 +Author: Alpar Torok +Date: Fri Nov 2 06:34:29 2018 +0200 + + Upadte TESTING.ascidoc BWC section (#35126) + + * Upadte BWC section + +commit f22700812e9d085673ee6f01f882b9d4ab04ac8d +Author: Alpar Torok +Date: Fri Nov 2 05:27:40 2018 +0200 + + Introduce build qualifier parameter (#35155) + + * Introduce property to set version qualifier + + - VersionProperties.elasticsearch is now a string which can have qualifier + and snapshot too + - The Version class in the build no longer cares about snapshot and + qualifier. + +commit dbc7c9259c8208cb71fd1c97ab2c9c6b7ffb7584 +Author: Nhat Nguyen +Date: Thu Nov 1 21:58:03 2018 -0400 + + CCR/TEST: Add debug log to testFailOverOnFollower + +commit 2a60c24043ad01f241efd4d4de2df354b437ffcd +Author: Andy Bristol +Date: Thu Nov 1 16:59:06 2018 -0700 + + [test] mute QueryProfilerIT.testProfileMatchesRegular + +commit b260406ecf6f05228e6e57e5217cc338a27932c8 +Author: Tal Levy +Date: Thu Nov 1 14:38:00 2018 -0700 + + ILM: use node _id attribute when allocating to one node (#35061) + + ILM's Shrink Action was using a nodes "_name" attribute to + allocate to prepare for the shrink step. Since the name is + configurable by a user and may use the same name for + multiple nodes on one machine, _id is safer since it is guaranteed + to be unique. + + closes #35043. + +commit f97c9b6af28cd32c49e45239cf18d203d282e5c7 +Author: lcawl +Date: Thu Nov 1 14:27:49 2018 -0700 + + [DOCS] Fixes location of beta markup + +commit eec357ebdea572b9cc27fe62d5a6b1a9a2c15528 +Author: Andy Bristol +Date: Thu Nov 1 13:01:53 2018 -0700 + + [test] quote base_path in repository tests + +commit fda173d7aa14692628fc7c33b1e1f05e339fe3eb +Author: Julie Tibshirani +Date: Thu Nov 1 12:41:07 2018 -0700 + + Add a note around using separate indices for percolator queries and documents. (#35109) + +commit 746d94e299ca135059e9de6a428b7d67d952cac9 +Author: Julie Tibshirani +Date: Thu Nov 1 12:02:55 2018 -0700 + + Unmute AbstractQueryTestCase#testToQuery. + + The RangeQueryBuilderTests#testToQuery failures were fixed in #34868 and #35145. + +commit 781130c7ab2841dbf867cb38f87d79eae16e9411 +Author: lcawl +Date: Thu Nov 1 11:50:30 2018 -0700 + + [DOCS] Fixes formatting of dynamic settings links + +commit c6c01425bbde09c7ec48a3cd3075382eb78cd534 +Merge: 6b312a500d2 8fb3290e5c4 +Author: Tal Levy +Date: Thu Nov 1 11:38:42 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 8fb3290e5c4fdbf31bb98d3e27e004c71f1d5664 +Author: Julie Tibshirani +Date: Thu Nov 1 11:15:26 2018 -0700 + + Fix a bug in function_score queries where we use the wrong boost_mode. (#35148) + +commit 0bdf6a768a9d3b70cce4fa93dbfdadb0464a19c9 +Author: Alpar Torok +Date: Thu Nov 1 19:46:58 2018 +0200 + + Make start/stop metods package protected + + As requested in #34818 + +commit ffacd6f0e4618d17dbcd432cfa350da6efb28491 +Author: Alpar Torok +Date: Thu Nov 1 19:41:41 2018 +0200 + + Testclusters: Refactor and fix for newly added tests (#34818) + + * Testclusters: Refactor and fix for newly added tests + * Move state management outside of the config object + +commit 928c9a12d218721258ab16cc7f9e606e27942072 +Author: Alpar Torok +Date: Thu Nov 1 19:31:52 2018 +0200 + + Mute failing tests #35173 + +commit 6b312a500d2db33730576496fdfd2150e9de9ecd +Author: Tal Levy +Date: Thu Nov 1 10:22:55 2018 -0700 + + uninherit from AbstractComponent in IndexLifecycleService + +commit c3cf7dd305b3d4636405c292659dbff05307f146 +Merge: f8e23f64004 f8378d91a62 +Author: Tal Levy +Date: Thu Nov 1 09:05:41 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 77fab62ebe18549f94279c0d476e9e12dd5efb05 +Author: Jake Landis +Date: Thu Nov 1 11:08:04 2018 -0500 + + ingest: add common options to each processor's documentation (#35091) + + * adds `if`, `on_failure`, `tag`, and `ignore_failure` to table for each processor + + part of #33188 + + * added ingore_failure + + * fix whitespace noise + +commit f8378d91a629550dad2fb388e0bec238d1dab279 +Author: Alpar Torok +Date: Thu Nov 1 17:43:57 2018 +0200 + + Implement VersionCollection in Java (#34050) + +commit 2fadec5c3d63a6615b83b2a287a78db1aba7522d +Author: Benjamin Trent +Date: Thu Nov 1 10:02:24 2018 -0500 + + ML: Add support for rollup Indexes in Datafeeds (#34654) + + * Adding rollup support for datafeeds + + * Fixing tests and adjusting formatting + + * minor formatting chagne + + * fixing some syntax and removing redundancies + + * Refactoring and fixing failing test + + * Refactoring, adding paranoid null check + + * Moving rollup into the aggregation package + + * making AggregationToJsonProcessor package private again + + * Addressing test failure + + * Fixing validations, chunking + + * Addressing failing test + + * rolling back RollupJobCaps changes + + * Adding comment and cleaning up test + + * Addressing review comments and test failures + + * Moving builder logic into separate methods + + * Addressing PR comments, adding test for rollup permissions + + * Fixing test failure + + * Adding rollup priv check on datafeed put + + * Handling missing index when getting caps + + * Fixing unused import + +commit 7b13d0592f1f969d13719bc5bc135bd558e80e0d +Author: Igor Motov +Date: Thu Nov 1 10:12:05 2018 -0400 + + Watcher: fix metric stats names (#34951) + + * Watcher: fix metric stats names + + The current watcher stats metric names doesn't match the current + documentation. This commit fixes the behavior of `queued_watches` + metric, deprecates `pending_watches` metric and adds `current_watches` + to match the documented behavior. It also fixes the documentation, which + introduced `executing_watches` metric that was never added. + + Fixes #34865 + +commit 7744f6f59044689a90de1f3d657b7a66192fbe3f +Author: Alpar Torok +Date: Thu Nov 1 14:46:36 2018 +0200 + + Fix checkstyle + +commit d46f3528c0f244e5a78f11c9a730d3f4be737c68 +Author: Itamar Syn-Hershko +Date: Thu Nov 1 12:57:06 2018 +0200 + + [Docs] Updating URL for Openstack Swift plugin (#34136) + + The repository plugin for Openstack Swift was developed originally by Wikimedia foundation but is now retired. + This changes the link to the repo where the actively maintained fork lives now. + +commit 14f540e8e66cdf6b3c7ee2a8fda9111f89dc9da8 +Author: Ke Li +Date: Thu Nov 1 18:06:51 2018 +0800 + + Deprecate unicodeSetFilter in favour of unicode_set_filter (#29215) + +commit b6659847c279f83dbc9bc3d250fbb113fb125993 +Author: Andrei Stefan +Date: Thu Nov 1 11:30:05 2018 +0200 + + SQL: handle wildcard expansion on incorrect fields (#35134) + +commit 4f78e2b5767936afafec6adbdff6a90c345e292a +Author: Marios Trivyzas +Date: Thu Nov 1 10:23:49 2018 +0100 + + SQL: Improve painless script generated from `IN` (#35055) + + Replace standard `||` and `==` painless operators with + new `in` method introduced in `InternalSqlScriptUtils`. + This allows the list of values to become a script variable + which is replaced each time with the list of values provided + by the user. + + Move In to the same package as InPipe & InProcessor + + Follow up to #34750 + + Co-authored-by: Costin Leau + +commit f3e150b0ea99640b552db51f5b8a3410ec4c7686 +Author: Alexandru Rusanescu +Date: Thu Nov 1 18:22:06 2018 +0900 + + [Docs] Update query_cache.asciidoc (#33340) + + Add note about non-visibility of cache content. + +commit e28509fbfea45bd381708a962ab4bdc04dda1a47 +Author: Nik Everett +Date: Wed Oct 31 21:23:20 2018 -0400 + + Core: Less settings to AbstractComponent (#35140) + + Stop passing `Settings` to `AbstractComponent`'s ctor. This allows us to + stop passing around `Settings` in a *ton* of places. While this change + touches many files, it touches them all in fairly small, mechanical + ways, doing a few things per file: + 1. Drop the `super(settings);` line on everything that extends + `AbstractComponent`. + 2. Drop the `settings` argument to the ctor if it is no longer used. + 3. If the file doesn't use `logger` then drop `extends + AbstractComponent` from it. + 4. Clean up all compilation failure caused by the `settings` removal + and drop any now unused `settings` isntances and method arguments. + + I've intentionally *not* removed the `settings` argument from a few + files: + 1. TransportAction + 2. AbstractLifecycleComponent + 3. BaseRestHandler + + These files don't *need* `settings` either, but this change is large + enough as is. + + Relates to #34488 + +commit b5e5e93c4671c55f348c5b53c1c10f5721147a66 +Author: Igor Motov +Date: Wed Oct 31 20:32:18 2018 -0400 + + Fixes randomDateTimeZone method (#35145) + + The randomDateTimeZone method shouldn't return deprecated timezones + this causes some tests to fail with deprecation warning. + +commit f8e23f64004fc478f7441ad0b7f535cc764a4917 +Author: Tal Levy +Date: Wed Oct 31 17:09:35 2018 -0700 + + update ILM integ test cluster poll interval to 1s (#35113) + +commit bfaa94f53e5b2cf03593427d33b9ee583229373b +Author: Ryan Ernst +Date: Wed Oct 31 16:31:14 2018 -0700 + + Packaging: Update procrun executables to version 1.1.0 (#35147) + + This commit updates the procrun manager and service exes to 1.1.0. There + are a few bug fixes, including for a bug which can cause lingering + processes when removing the service. + +commit 9ef4788c13349a055145996f6bbc3d8ef170e07f +Author: Seong-hyun, Oh +Date: Thu Nov 1 06:15:46 2018 +0900 + + Make XContentBuilder in AliasActions build `is_write_index` field (#35071) + + Make XContentBuilder in AliasesActions build `is_write_index` field + +commit d181d1bab16e75f4c8528d9ac641f26e67ea75ea +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Wed Oct 31 22:11:59 2018 +0100 + + Remove deprecated url parameters `_source_include` and `_source_exclude` (#35097) + + Removes `_source_include` and `_source_exclude` url parameters. + These parameters have been deprecated in #33475. + + Closes #22792 + +commit f87a53451bae56c1a685c426373c6e9598bcb011 +Author: Costin Leau +Date: Wed Oct 31 23:07:36 2018 +0200 + + SQL: Register missing processors (#35121) + + Add registration (and tests) for missing processors in the serialization + chain. + + Fix #35119 + +commit e6f9f0666e2a2cfcd45ef33adb91c49bfc80ba4d +Author: Armin Braun +Date: Wed Oct 31 21:33:49 2018 +0100 + + NETWORKING: MockTransportService Wait for Close (#35038) + + * NETWORKING: MockTransportService Wait for Close + + * Make `MockTransportService` wait `30s` for close listeners to run before failing the assertion + * Closes #34990 + +commit 0072c90e2a6120d713c80f5380a099511ecf1adf +Author: David Turner +Date: Wed Oct 31 19:21:24 2018 +0000 + + Pre-populate unicast hosts files (#35136) + + Today when ESIntegTestCase starts some nodes it writes out the unicast hosts + files each time a node starts its transport service. This does mean that a + number of nodes can start and perform their first pinging round without any + unicast hosts which, if the timing is unlucky and a lot of nodes are all + started at the same time, can lead to a split brain as in #35052. + + Prior to #33554 this was unlikely to happen since the MockUncasedHostsProvider + would always have yielded the existing hosts, so the timing would have to have + been implausibly unlucky. Since #33554, however, it's more likely because the + race occurs between the start of the first round of pinging and the writing of + the unicast hosts file. It is realistic that new nodes will be configured with + the existing nodes from startup, so this change reinstates that behaviour. + + Closes #35052. + +commit 5f4b23f8c140bff31f79062782d4ee8ef74c7f41 +Author: Tal Levy +Date: Wed Oct 31 11:49:29 2018 -0700 + + cleanup ILM qa structure (#35110) + + This commit does a few things + + - moves ILM-specifc rest yaml tests into plugin/ilm/qa, and creates special + :plugin:ilm:qa:rest module to test them + - removes the with-security tests of the yaml tests since they are covered in + the rest tests now + - moves ChangePolicyforIndexIT into the qa/multi-node project since that test is + not currently running in main ilm since integTest is disabled + +commit 6492eaa84d4efe2af1cd21725d2e59f1a2945558 +Author: Andy Bristol +Date: Wed Oct 31 11:48:58 2018 -0700 + + [test] mad tests more lenient approximation + +commit ca620ff4ceddc7ec3817b6e240a2a28c2e2e60f2 +Author: Nik Everett +Date: Wed Oct 31 14:38:29 2018 -0400 + + Loggers: Drop last deprecated logger function (#35082) + + Drop the last function from `Loggers` that just wraps Log4j2. + + Relates to #32174 + +commit 010845b2335d031b80589044dc5f23ac48b2e631 +Author: Andy Bristol +Date: Wed Oct 31 11:36:25 2018 -0700 + + [test] dont skip last version for mad + +commit a294a7c6b508badf2a02feffc22a34ee550d676a +Author: Tal Levy +Date: Wed Oct 31 11:17:10 2018 -0700 + + fix IndexLifecycleService setting member + + the settings variable was previously created by + the AbstractComponent class inherited by IndexLifecycleService. + this is no more. + +commit f27251dfcb2dadb897ab5c0e80a1d70827f501ac +Author: Qian Hong +Date: Thu Nov 1 05:04:48 2018 +1100 + + [Docs] Improve Redindex/Update/Delete By Query. (#35127) + + Fix a copy paste error and improve formatting in + Redindex/Update/Delete By Query and fix formula in Update By Query. + +commit d5d28420b636275705d7998a88592b4060b905db +Merge: e540abd5514 ef5181c6784 +Author: Tal Levy +Date: Wed Oct 31 10:47:07 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit ef5181c6784ab8022563bd5c6cb5eb6ba064b07e +Author: Luca Cavanna +Date: Wed Oct 31 17:32:53 2018 +0100 + + Allow to enable pings for specific remote clusters (#34753) + + When we connect to remote clusters, there may be a few more routers/firewalls in-between compared to when we connect to nodes in the same cluster. We've experienced cases where firewalls drop connections completely and keep-alives seem not to be enough, or they are not properly configured. With this commit we allow to enable application-level pings specifically from CCS nodes to the selected remote nodes through the new setting `cluster.remote.${clusterAlias}.transport.ping_schedule`. The new setting is similar `transport.ping_schedule` but it does not affect intra-cluster communication, pings are only sent to specific remote cluster when specifically enabled, as they are disabled by default. + + Relates to #34405 + +commit 72cb885abc7fac8a56ff8281b50ef68f4fa83c8d +Author: Lisa Cawley +Date: Wed Oct 31 09:18:14 2018 -0700 + + [DOCS] Updates monitoring diagram (#35096) + +commit 0b8465f2c01a8ac17959718f4334c799951d2bfa +Author: Mayya Sharipova +Date: Wed Oct 31 12:06:48 2018 -0400 + + Update type to "seat" (#35086) + + otherwise this throws an exception: java.lang.IllegalArgumentException: Rejecting mapping update to [seats] as the final mapping would have more than 1 type: [seat, _doc] + + May be, later for the types removal, we can modify `seats.json` to have a type `_doc` instead of `seat` + +commit 1ce3c92a2d4d2c387011f47628de346da2413271 +Author: Sue Gallagher <36747279+Sue-Gallagher@users.noreply.github.com> +Date: Wed Oct 31 07:16:36 2018 -0700 + + [DOCS] Add info on calendar vs fixed interval. (#31638) + + Extensive edit to add additional information on the difference between calendar intervals and fixed-length intervals. + +commit bdd84609063b69e5da5f3e2dbdb1c31a451122da +Author: Albert Zaharovits +Date: Wed Oct 31 14:49:33 2018 +0200 + + HLRest: add security authenticate API (#33552) + + This adds the security `_authenticate` API to the HLREST client. + It is unlike some of the other APIs because the request does not + have a body. + The commit also creates the `User` entity. It is important + to note that the `User` entity does not have the `enabled` + flag. The `enabled` flag is part of the response, alongside + the `User` entity. + Moreover this adds the `SecurityIT` test class + (extending `ESRestHighLevelClientTestCase`). + + Relates #29827 + +commit 28bb1a3ac0c529beb38cd3553b50e095a07df13b +Author: Vladimir Dolzhenko +Date: Wed Oct 31 13:48:35 2018 +0100 + + XPack/HLRC request/response compatibility tests (#34547) + + + Relates #34451 , #29827 + +commit 674225aaa1474c7a28ce8505042a90588ee29379 +Author: Luca Cavanna +Date: Wed Oct 31 13:07:02 2018 +0100 + + [TEST] Enforce skip headers when needed (#34735) + + The java yaml test runner supports sending request headers, yet not all clients support headers. This commit makes sure that we enforce adding a skip section with feature "headers" whenever headers are used in a do section as part of a test. That decreases the chance for new tests to break client builds due to the missing skip section. + + Closes #34650 + +commit 00dc2ba36f6c2f97d0967108c69a42973f51340f +Author: Dimitris Athanasiou +Date: Wed Oct 31 10:55:11 2018 +0000 + + [ML] Enable reusing field extraction logic when no time field is required (#35100) + +commit bf6f521827d4784a22313aee0fc7ea5823ff03d5 +Author: Marios Trivyzas +Date: Wed Oct 31 10:32:16 2018 +0100 + + SQL: Fix NPE thrown if HAVING filter evals to null (#35108) + + Surround script with `nullSafeFilter` also for `AggFilter` translation. + Fixed query translation tests to properly set `onAggs`. + + Fixes: #35107 + +commit 3fa67c5d8a7a9fc588582e0d9bb89299f300dfae +Author: Armin Braun +Date: Wed Oct 31 07:52:37 2018 +0100 + + DISCOVERY: Cleanup AbstractDisruptionTestCase (#34808) + + * DISCOVERY: Cleanup AbstractDisruptionTestCase + + * Make the internal test cluster manage minimum master nodes where we used the default of (nodes / 2 + 1) before + * Remove use of the `NodeConfigurationSource` indirection + * Relates #33675 + +commit e540abd551404ce8b76f49209c113265138db27b +Author: debadair +Date: Tue Oct 30 18:27:43 2018 -0700 + + [DOCS] Fixed edit links for ILM APIs and added the APIs to the REST ref (#35070) + + * [DOCS] Fixed edit links for ILM APIs and added the APIs to the REST API section. + + * [DOCS] Fixed link to ILM APIs. + +commit fdb0db30c4a3d79195e8e13941c13e2c6710e008 +Author: Tal Levy +Date: Tue Oct 30 16:42:12 2018 -0700 + + throw exception when ilm-rollover-step runs on index without rollover alias (#35065) + + The ILM Rollover Step can execute on the incorrect index if the rollover alias + exists on another valid index, but not the one the step is executing against. This + is a problem and is now guarded against + +commit 5141084048a50b0110e48b34fff1053e2ab08cc9 +Author: Tal Levy +Date: Tue Oct 30 16:19:05 2018 -0700 + + rename CRUD api REST path prefix _ilm to _ilm/policy (#35056) + + This PR renames the CRUD APIS for ILM + + GET _ilm/, _ilm -> _ilm/policy/, _ilm/policy + PUT _ilm/ -> _ilm/policy/ + DELETE _ilm/ -> _ilm/policy/ + + closes #34929. + +commit 455906d431616ba6e62cefed4cd8c4a3f691cc7d +Author: Marios Trivyzas +Date: Tue Oct 30 22:31:29 2018 +0100 + + SQL: [tests] Remove AbstractBuilderTestCase dep (#35099) + + Extend directly from `EsTestCase` to speed up execution. + + Fixes: #35098 + +commit 086ada4c08905724b96fb74031e61978e0b458da +Author: Nik Everett +Date: Tue Oct 30 16:10:38 2018 -0400 + + Core: Drop settings member from AbstractComponent (#35083) + + Drops the `Settings` member from `AbstractComponent`, moving it from the + base class on to the classes that use it. For the most part this is a + mechanical change that doesn't drop `Settings` accesses. The one + exception to this is naming threads where it switches from an invocation + that passes `Settings` and extracts the node name to one that explicitly + passes the node name. + + This change doesn't drop the `Settings` argument from + `AbstractComponent`'s ctor because this change is big enough as is. + We'll do that in a follow up change. + +commit f5323cd86ed96fbf964445440c2c98165c041af9 +Author: Tal Levy +Date: Tue Oct 30 12:54:02 2018 -0700 + + rename ILM Phase "minimum_age" to "min_age" (#35058) + + closes #34927. + +commit 0f9a7b33440871dec943dda798693169cabb216c +Author: Nik Everett +Date: Tue Oct 30 15:47:25 2018 -0400 + + Docs: Fix build after moving SQL QA + + I moved SQL's QA in #34496 to make it easier to run all of SQL tests in + one go but forgot to update the docs build. This fixes that. + +commit 09a124e8d6a2b9e0ae66e78b73e6483c74cb99f4 +Author: Jay Modi +Date: Tue Oct 30 13:35:07 2018 -0600 + + Native roles store uses mget to retrieve roles (#33531) + + The native roles store previously would issue a search if attempting to + retrieve more than a single role. This is fine when we are attempting + to retrieve all of the roles to list them in the api, but could cause + issues when attempting to find roles for a user. The search is not + prioritized over other search requests, so heavy aggregations/searches + or overloaded nodes could cause roles to be cached as missing if the + search is rejected. + + When attempting to load specific roles, we know the document id for the + role that we are trying to load, which allows us to use the multi-get + api for loading these roles. This change makes use of the multi-get api + when attempting to load more than one role by name. This api is also + constrained by a threadpool but the tasks in the GET threadpool should + be quicker than searches. + + See #33205 + +commit 6ecb8ff34498c8cc04fd46ea8ccc36c387722fcd +Author: Gordon Brown +Date: Tue Oct 30 13:33:32 2018 -0600 + + Move to Error step if ClusterState* steps throw (#35069) + + Previously, if ClusterStateActionSteps or ClusterStateWaitSteps threw an + exception executing, the exception would only be caught and logged by + the generic ClusterStateUpdateTask machinery and the index would become + stuck on that step. + + Now, exceptions thrown in these steps will be caught and the index will + be moved to the Error step. + +commit 7de9eda3916458aa02a4c83e5f7788cfcfa3574b +Author: Nik Everett +Date: Tue Oct 30 15:14:33 2018 -0400 + + Graph: Move qa into plugin (#34517) + + Move `x-pack/qa/smoke-test-graph-with-security` to + `x-pack/plugin/graph/qa/security` which should make it easier to run all + of the tests with a single command. It also lines up the directories + more closely with newer projects like cross cluster replication. + +commit 6857d305270be3d987689fda37cc84b7bc18fbb3 +Author: Nik Everett +Date: Tue Oct 30 14:54:15 2018 -0400 + + SQL: Move qa directory into sql directory (#34496) + + Moves `x-pack/qa/sql/*` into `x-pack/plugin/sql/qa` to make it simpler + to run all of the sql tests. This lines up with how newer projects like + cross cluster replication are testing themselves. + +commit 6abddd8ca614de9d47bd79c342113461b4b37526 +Author: Costin Leau +Date: Tue Oct 30 20:17:20 2018 +0200 + + SQL: Optimizer rule for folding nullable expressions (#35080) + + Add optimization for folding nullable expressions with a NULL + argument. This is a variant of folding for the NULL case. + + Fix #34826 + +commit 512319cef7ef36d102c5a47505e4265fae6de2ea +Author: Ryan Ernst +Date: Tue Oct 30 11:15:34 2018 -0700 + + Test: Filter out deprecated joda tzs in tests (#34868) + + This commit filters out usage of deprecated tzs by tests. These are + tested separately and should not require checking for warnings on any + test using random timezones. + + closes #34188 + +commit 2b2e208c4c0103f9fe1a13b032a90bbe83ebec65 +Author: Nik Everett +Date: Tue Oct 30 14:12:59 2018 -0400 + + Docs: Remove range notation from random score docs (#35093) + + The `random_score` function produces values between 0 (inclusive) and 1 + (exclusive) and documented it with fancy methematical range notation. It + is so fancy I thought it was a typo. This changes the documentation to + use words. + + Relates to #35084 + +commit f9dd33a0b9c5f238c39d48a1513c5e4f67241c14 +Author: Zachary Tong +Date: Tue Oct 30 13:50:50 2018 -0400 + + [Rollup] Proactively resolve index patterns in RollupSearch endoint (#34930) + + This changes the RollupSearch endpoint to proactively resolve index + patterns. If the index pattern(s) match more than one rollup index, + an exception is throw as before. But if the pattern only matches one + rollup index, execution is allowed to continue (unlike before where + it would assume all patterns were for raw data). + + This also allows the search endpoint to resolve aliases that point to + a rollup index. + + Also tweaks the documentation to make this clear. + + Closes #34828 + +commit 70da490f34d279919da22362164d64740a50c86d +Author: Julie Tibshirani +Date: Tue Oct 30 10:19:12 2018 -0700 + + Remove some documentation that only makes sense with multiple types. (#35066) + + * Remove a tip about ignore_above that only makes sense with multiple types. + * Remove a line from the percolator documentation that refers to multiple types. + +commit be75b40a2945a6f34080165580cc14263f435362 +Author: Vladimir Dolzhenko +Date: Tue Oct 30 18:00:14 2018 +0100 + + Fix LineLength Check Suppressions: index.mapper (#35087) + + Relates #34884 + +commit 70e939ee3825ec603273ec4c3e391bc64b422b0b +Author: Nik Everett +Date: Tue Oct 30 10:34:03 2018 -0400 + + Search: Test deprecated field in mixed cluster + + Update the test for the deprecated `_source_exclude` parameter to run + against 6.6.0+ now that we've backported the deprecation. + + Relates to #33475 + +commit 18c72e86c57c7f719a4e99e98cc75c9ae3e92a69 +Merge: 6cf7f1205af 7ef65dedc36 +Author: Tal Levy +Date: Tue Oct 30 08:09:57 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 6cf7f1205afd494077e1054aadbf30a9750ac5b7 +Author: Tal Levy +Date: Tue Oct 30 08:06:53 2018 -0700 + + Revert "add ILM rolling upgrade tests (#32828)" + + This reverts commit 5c88356ad6643c2b588f36cf36ed80d423d3838a. + +commit 7ef65dedc36735a0e84f482bd9fbc4acab9f7a17 +Author: Luca Cavanna +Date: Tue Oct 30 16:06:31 2018 +0100 + + [TEST] improve validation of yaml suites (#34957) + + Validation of test sections and suites consists of checking that the proper skip features sections are in place depending on the features used in tests. + + The validation logic was previously only performed on do sections included in each test section, and the skip needed to be present in the same test section. What happens often though is that the skip is added to the setup section, or the teardown section. + + This commit improves the validation of test suites by validating setup and teardown section first, then looking at each test section while still eventually reading the skip section from setup or teardown. + + We are also making SkipSection, SetupSection, TearDownSection, ClientYamlTestSection and ClientYamlTestSuite immutable. Previously it was possible to utilize constants like SetupSection.EMPTY, which were modifiable and affect every other future users by modifiying them. This has been corrected. + + Also, validation has been improved to cumulate errors so that all the errors from a suite will be listed at once. + + Relates to #34735 + +commit b8280ea7cceb5f31c2827f09a49b18eaeb9ddebd +Author: Andy Bristol +Date: Tue Oct 30 07:22:52 2018 -0700 + + median absolute deviation agg (#34482) + + This commit adds a new single value metric aggregation that calculates + the statistic called median absolute deviation, which is a measure of + variability that works on more types of data than standard deviation + + Our calculation of MAD is approximated using t-digests. In the collect + phase, we collect each value visited into a t-digest. In the reduce + phase, we merge all value t-digests, then create a t-digest of + deviations using the first t-digest's median and centroids + +commit 7bd113d2848df9880c138e5de812980c5506d4e4 +Author: Andrei Stefan +Date: Tue Oct 30 16:18:52 2018 +0200 + + Test for https://github.com/elastic/elasticsearch/issues/30054 bug (#35050) + + Added a test for a bug in Elasticsearch (https://github.com/elastic/elasticsearch/issues/32941) that had implications in SQL and activated another test that didn't work in the past + +commit 253152172c8ba1a17884ef4c19de128d4653242d +Author: Andrei Stefan +Date: Tue Oct 30 15:52:05 2018 +0200 + + SQL: Proper handling of nested fields at the beginning of the columns list (#35068) + +commit cc4dd988720d8ac2898046cd1f7c3f1aded124c2 +Author: Nik Everett +Date: Tue Oct 30 09:14:07 2018 -0400 + + Docs: Fix build for java-api + + The java-api docs had some broken references. Mostly because it is + difficult to build them so they are difficult to check. + + Relates to #34528 + +commit 502745f8e25da0c5fd5aa5d9c4425b7f386b0eef +Author: Colin Goodheart-Smithe +Date: Tue Oct 30 12:46:39 2018 +0000 + + Adds backport label to list to ignore (#35079) + +commit cf24231b1bcc3b71ac8e43e1020e61496a169c6d +Author: Alan Woodward +Date: Tue Oct 30 11:43:43 2018 +0000 + + Add missing full stop in Bootstrap checks doc (#35078) + + Fixes #35075 + +commit 97f74c5a3893186e340527817ec2d14c71ab3abb +Merge: 7a3cd10718d e0fef52058e +Author: Andrey Ershov +Date: Tue Oct 30 13:13:38 2018 +0300 + + Merge branch 'master' into 'zen2' + + Conflicts during the merge: + 1. >=140 chars line length fixed for a lot of project files and warnings + for those files are no longer suppressed + 2. Node name is removed from AbstractComponent, it’s no longer taken + from settings, but is explicitly passed as constructor argument and + there were quite a few new classes on zen2 branch that require this + change + 3. TransportResponseHandler interface changed (new method added) and + Zen2 makes a lot of subclasses in tests + 4. Deprecated way of obtaining logger was changed + +commit c74232037aa902b7b1bcc6d554d7d1b1d72c06cd +Author: Alan Woodward +Date: Tue Oct 30 10:27:31 2018 +0000 + + Remove Accountable interface from BytesReference (#34900) + +commit 6da2fb7d5b65ce5201d463553cecd89723078ea3 +Author: Martijn van Groningen +Date: Tue Oct 30 11:03:30 2018 +0100 + + Change CCR API request classes to use Writeable serialization instead of Streamable (#34911) + + Only the follow stats request couldn't be changed to use Writeable serialization, + because that requires changes in `TransportTasksAction` and `BaseTasksRequest` base classes. + +commit 5e0c9b7c1ad44b92d5c634f5d4f4426f17864079 +Author: markwalkom +Date: Tue Oct 30 20:22:42 2018 +1100 + + Update reindex.asciidoc (#35062) + + Highlighted that `_source` needs to exist in the source index. + +commit e0fef52058ee842a3639b3be6bc7d02ad7b91bec +Author: Marios Trivyzas +Date: Tue Oct 30 10:12:33 2018 +0100 + + SQL: Fix failing test due to random timezone (#35067) + + Use null values for `languages` instead of `birth_date` to avoid + differences with H2 because of timezone randomization. + + Fixes: #35042 + +commit f7760ddacb2f2600e2c0de23ca7b218c63913825 +Author: David Turner +Date: Tue Oct 30 08:45:23 2018 +0000 + + Increase discovery log level in NetworkDisruptionIT + + Sometimes the cluster forming here will split-brain when it grows up to 5 + nodes. This could be a timing issue or could be something going wrong in + discovery, so this asks for more logs. Relates #35052 + +commit 995bf0ee668282fbde420eaf2eb028af75a74d12 +Author: Przemyslaw Gomulka +Date: Tue Oct 30 09:08:12 2018 +0100 + + Bulk Api support for global parameters (#34528) + + Bulk Request in High level rest client should be consistent with what is + possible in Rest API, therefore should support global parameters. Global + parameters are passed in URL in Rest API. + + Some parameters are mandatory - index, type - and would fail validation + if not provided before before the bulk is executed. + Optional parameters - routing, pipeline. + + The usage of these should be consistent across sync/async execution, + bulk processor and BulkRequestBuilder + + closes #26026 + +commit d67c88fa120df8ced0dfa7e2304884e44aed40d7 +Author: Ken MacInnis +Date: Tue Oct 30 18:58:32 2018 +1100 + + [Docs] Update update-settings.asciidoc (#35073) + + Add explanation for ?include_defaults parameter and link to + Cluster Get Settings reference + +commit 5dda2b0c7aa9b4fec08072afb9d64c8195deef1f +Author: Ryan Ernst +Date: Mon Oct 29 19:45:35 2018 -0700 + + Remove remaining line length violations in o.e.cluster (#34941) + + relates #34923, #34884 + +commit d85a654ebbdacb86ac0c51bf541d391d61a49ebd +Author: Dimitris Athanasiou +Date: Mon Oct 29 22:56:53 2018 +0000 + + [ML] Refactor doc value format into ExtractedField (#35053) + + This commit moves the knowledge of which doc value format + to be used down to the `ExtractedField` instead of being + in the data extractor. + +commit 794d4fa87933e461baeca0aa6f6969cb23039860 +Author: Gordon Brown +Date: Mon Oct 29 15:13:14 2018 -0600 + + Label required scripts in Scripted Metric Agg docs (#35051) + + When combine_script and reduce_script were made into required + parameters for Scripted Metric aggregations in #33452, the docs were + not updated to reflect that. This marks those parameters as required + in the documentation. + +commit c9e4d26a531f213094b89b6956d2dddbbb5a6567 (es/index-lifecycle) +Merge: f6ac0e4bbca f1135ef0ceb +Author: Tal Levy +Date: Mon Oct 29 14:03:55 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit f1135ef0ceb9c8d74ba4589c285d3cb05e908bcb +Author: Pratik Sanglikar +Date: Mon Oct 29 12:52:30 2018 -0700 + + Core: Replace deprecated Loggers calls with LogManager. (#34691) + + Replace deprecated Loggers calls with LogManager. + + Relates to #32174 + +commit 6169e9345e282b6c1986dfff192656b7bb993e3b +Author: Spencer +Date: Mon Oct 29 12:09:53 2018 -0700 + + [REST-API] fix url for reload_secure_settings + +commit 3051e37e45e1ba823d903d8259e8eb80188ef8ec +Author: Nik Everett +Date: Mon Oct 29 13:58:55 2018 -0400 + + Docs: Document refresh for reindex and friends + + Reindex and friends don't support `refresh=wait_for` and this documents + that. + +commit f6ac0e4bbca339d4e6d7db9a6b8ee221ccc2a702 +Author: Gordon Brown +Date: Mon Oct 29 11:18:12 2018 -0600 + + [ILM] Fix Move To Step API causing ILM to hang (#34618) + + The Move To Step API now checks to see if the target step is an + AsyncActionStep, and if so, runs it. + + Previously, AsyncActionSteps would only be run when they are entered by + executing the previous step, so if an AsyncActionStep was entered via + the Move To Step API, ILM would never touch that index again. + +commit c9a8c58039f881ddf37b6209f3cdf7b73706814b +Author: Alpar Torok +Date: Mon Oct 29 18:13:17 2018 +0200 + + Fix test failure on Windows (#35047) + +commit 6df1c9e818101f023c01daec064a7cf9d304197c +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Mon Oct 29 17:06:38 2018 +0100 + + Deprecate `_source_include` and `_source_exclude` url parameters (#33475) + + Deprecates `_source_include` and `_source_exclude` url parameters + in favor of `_source_inclues` and `_source_excludes` because those + are consistent with the rest of Elasticsearch's APIs. + + Relates to #22792 + +commit 42b4f23a76762b150ba9f40e058e59de24278241 +Author: Marios Trivyzas +Date: Mon Oct 29 16:27:07 2018 +0100 + + SQL: Add `CAST` and `CONVERT` to `SHOW FUNCTIONS` (#34940) + + Fixes: #34939 + +commit 6fd7ea71f1f8aa9e058da9528fdaca40a7046d9e +Author: Nik Everett +Date: Mon Oct 29 10:32:17 2018 -0400 + + HLRC: DRY up remaining CRUD docs (#34925) + + This further applies the pattern set in #34125 to reduce copy-and-paste + in the multi-document CRUD portion of the High Level REST Client docs. + It also adds line wraps to snippets that are too wide to fit into the box + when rendered in the docs, following up on the work started in #34163. + +commit b093116a1e82e8e06c3ae61dc841a30304193785 +Author: Nik Everett +Date: Mon Oct 29 10:05:24 2018 -0400 + + Logging: Drop another deprecated Loggers method (#34520) + + Drop a method from `Loggers` that we deprecated because it just + delegated to `LogManager`. + +commit cc9894d78a0b993a6d8fcb323a8329389ec7826b +Author: Jason Tedor +Date: Mon Oct 29 09:50:13 2018 -0400 + + Fix name of CCR stats transport action + + This class name should include an indication that it is for CCR. This + commit adds that to the name of this class. + +commit 26f5c509afa6ce836a0661e4666c869c779d1e4d +Author: Jason Tedor +Date: Mon Oct 29 09:37:13 2018 -0400 + + Fix CCR API specification (#34963) + + This commit fixes two issues with the CCR API specification: + - remove the CCR stats endpoint, it is not currently implemented + - fix the documentation links + +commit 329a94be0c16975df61d88dc8548e77a85661b00 +Author: Mark Tozzi +Date: Mon Oct 29 09:22:20 2018 -0400 + + Cleanup suppressed overlength line for action.support package (#34889) + + Clean up lines over 140 characters in the `org.elasticsearch.action.support.*` packages + + Relates to #34884 + +commit baa144e8441cd40dace467630b5de8725e41e2cf +Author: Alpar Torok +Date: Mon Oct 29 14:54:22 2018 +0200 + + Enforce a [skip] when using [contains] (#34840) + + Be friendly to other runners + +commit b2daaf15d11559456ef555f674e05a2e3e2b97c5 +Author: Martijn van Groningen +Date: Mon Oct 29 13:42:47 2018 +0100 + + [CCR] move tests that modify test cluster from the main test class to + a dedicated class. + +commit 69fe9a1bf118e248e2eb10cb936f60f363d9f1a5 +Author: Vladimir Dolzhenko +Date: Mon Oct 29 13:38:56 2018 +0100 + + HLRC XPack Protocol clean up: Migration; Graph; Watcher (#34639) + + Move parts related to XPack for packages Migration; Graph; Watcher from protocol to client + + Drop protocol module + + Relates #34451 + +commit 8d7889d20c9a7fa5307dec937da9fe1b2c22fb05 +Author: Marios Trivyzas +Date: Mon Oct 29 13:06:30 2018 +0100 + + SQL: Implement CAST between STRING and IP (#34949) + + This allows comparison between IP fields and STRING input + from the user where automatic conversion is not yet supported (e.g.: for IN) + + Closes: #34799 + +commit 731e48beff83caac621ea403773a3fee1ce8b21a +Author: Marios Trivyzas +Date: Mon Oct 29 12:02:28 2018 +0100 + + SQL: [Test] Fix and enable test with randomness (#34850) + + Increase minimum number of elements for List<> ctor arguments + for specific classes that validate the size of the list. + + Fixes: #34754 + +commit c9ae1928e0a1bbbdd86634dc2c055b924abe4019 +Author: Marios Trivyzas +Date: Mon Oct 29 11:32:17 2018 +0100 + + SQL: Fix function args verification and error msgs (#34926) + + Extract data type verification for function arguments to a single place + so that NULL type can be treated as RESOLVED for all functions. Moreover + this enables to have consistent verification error messages for all functions. + + Fixes: #34752 + Fixes: #33469 + +commit 389910f11dd01ad00d2c848bead54c5998b6a127 +Author: Christoph Büscher +Date: Mon Oct 29 10:23:07 2018 +0100 + + [Docs] Add migration note about expanded fields limit (#34920) + + Adds a note to warn users about the limit introduced in #26541. + +commit c455be7bc22619c86565b9dbd9ab56d35d2b484f +Author: David Roberts +Date: Mon Oct 29 10:06:12 2018 +0100 + + [ML] Rename the json file structure to ndjson (#34901) + + The file structure finder endpoint can find the NDJSON + (newline-delimited JSON) file format, but called it + `json`. This change renames the `format` for this file + structure to `ndjson`, which is more precise and will + hopefully avoid confusion. + +commit f13d5294488399dedccf2ad2d494de96539e2c77 +Author: Martijn van Groningen +Date: Mon Oct 29 09:29:13 2018 +0100 + + Fixed ccr docs broken links. + +commit 3a464bdee8bf2f5166d2dc43f1b53ebf87bae3c2 +Author: Christoph Büscher +Date: Mon Oct 29 09:28:11 2018 +0100 + + Add start rollup job support to HL REST Client (#34623) + + This change adds support for starting a rollup job to High Level REST Client. + + Relates to #29827 + +commit 3ef1fa566b292f884511abc35bd599a9ef85fcf1 +Author: vboulaye +Date: Mon Oct 29 08:06:34 2018 +0100 + + convert FilePermissionsTask.groovy to .java (#34674) + +commit 18015185271a32063a88ae34f6643bd1aac14014 +Author: Martijn van Groningen +Date: Mon Oct 29 07:45:27 2018 +0100 + + [CCR] Refactor stats APIs (#34912) + + * Changed the auto follow stats to also include follow stats. + * Renamed the auto follow stats api to stats api and changed its url path + from `/_ccr/auto_follow/stats` `/_ccr/stats`. + * Removed `/_ccr/stats` url path for the follow stats api, which makes + the index parameter a required parameter. + * Fixed docs. + +commit bad5972f62bf82e56c4b4d89f4d6ed0af626f2f0 +Author: Martijn van Groningen +Date: Mon Oct 29 07:38:55 2018 +0100 + + [CCR] Fix request serialization bug (#34917) + + and some parameters that were not set in tests. + +commit 9c27b407f09a7ef8ae26fd60584e06fe68e1dc91 +Author: Tim Vernum +Date: Mon Oct 29 17:17:56 2018 +1100 + + HLRC: Add security Create Token API (#34791) + + This adds the Create Token API (POST /_xpack/security/oauth2/token) + to the High Level Rest Client. + + Relates: #29827 + +commit bb5b59004e83aa22aac9866df0e27c48f7ecc53f +Author: Tim Vernum +Date: Mon Oct 29 11:23:36 2018 +1100 + + [TEST] Manually trigger resource watching (#34893) + + SSLTrustRestrictionsTests.testRestrictionsAreReloaded checks that the + SSL trust configuration is automatically updated reapplied if the + underlying "trust_restrictions.yml" file is modified. + + Since the default resource watcher frequency is 5seconds, it could + take 10 second to run that test (as it waits for 2 reloaded). + + Previously this test set that frequency to a very low value (3ms) so + that the elapsed time for the test would be reduced. However this + caused other problems, including that the resource watcher would + frequently run while the cluster was shutting down and files were + being cleaned up. + + This change resets that watch frequency back to its default (5s) and + then manually calls the "notifyNow" method on the resource watcher + whenever the restrictions file is modified, so that the SSL trust + configuration is reloaded at exactly the right time. + + Resolves: #34502 + +commit dc5bfe3a00b702c4df5d3bbd6cae1aa9ec1bb884 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Oct 29 11:15:42 2018 +1100 + + [DOCS] Fix typo in get role mappings API (#34952) + +commit a5ee134c40dd01e2b07008e8ff0211bb76017d2e +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Oct 29 10:12:13 2018 +1100 + + [HLRC] Add support for get role mappings API (#34637) + + This commit adds support for get role mappings API + in HLRC. + +commit 4e5c3052251babc00094f1444a5eae542270cec0 +Author: Nhat Nguyen +Date: Sun Oct 28 18:51:48 2018 -0400 + + Mute testRelocationWithConcurrentIndexing + + Tracked at #34950 + +commit 14fb7edc1d62a1f1ce0c9f384ca82fbb64372b12 +Author: Costin Leau +Date: Sun Oct 28 22:13:04 2018 +0200 + + SQL: Fix incorrect AVG data type (#34948) + + AVG now returns double (not the type of the target field) + + Close #33773 + +commit 71dddfd8f812ca4584deebf9db0e924685b0a597 +Author: Armin Braun +Date: Sat Oct 27 14:54:46 2018 +0200 + + MINOR: Remove Deadcode in NioTransport CORS (#34324) + + * Same as #34134 but for nio transport + +commit da612661d1663acd142a57e186f170529e704b79 +Author: Armin Braun +Date: Sat Oct 27 08:26:18 2018 +0200 + + NETWORK: Align Behaviour of NettyChan setSoLinger (#34870) + + * All other implementations only set SO_LINGER if the channel is still open and throw `IOException` on failure, the Netty one should do the same + +commit 277f49b9ca23bd89c5d8b45fa827c542e3100e11 +Author: Nik Everett +Date: Fri Oct 26 20:42:18 2018 -0400 + + HLRC: Drop extra level from user parser (#34932) + + It had support for an object between `created` and the root that we + don't return in 7.0+. + +commit 5c88356ad6643c2b588f36cf36ed80d423d3838a +Author: Tal Levy +Date: Fri Oct 26 15:40:26 2018 -0700 + + add ILM rolling upgrade tests (#32828) + +commit 544c2209420e2fa7e319f022743956342c355aa1 +Author: debadair +Date: Fri Oct 26 15:32:15 2018 -0700 + + [DOCS] Added attributes for HLRC javadoc packages. (#34934) + +commit 01c62fc06b50b2335f870c8b2e5fe7473868aeb5 +Author: Igor Motov +Date: Fri Oct 26 18:13:09 2018 -0400 + + Fix line length for bootstrap/client/discovery/gateway files (#34905) + + Removes the checkstyle suppressions for files in + org.elasticsearch.bootstrap/client/discovery/gateway packages. + + Relates to #34884 + +commit 052dfa5646592ffbdcfd5ede2aaf68e4de0baf1a +Author: Benjamin Trent +Date: Fri Oct 26 16:44:12 2018 -0500 + + HLRC: Adding Update datafeed API (#34882) + + * HLRC: Adding Update datafeed API + + * Addressing unused import + + * Adjusting docs and fixing minor comments + + * fixing comment + +commit 11fa8d37440403169a50b37dcc4a6d751e10ae69 +Author: Jake Landis +Date: Fri Oct 26 16:14:04 2018 -0500 + + Enforce 140 char line lengths for packages action.bulk/delete/explain/get/index (#34885) + + part of #34884 + +commit f5200e34ad6c3614457b5fe6577b71268922c2a8 +Author: Ryan Ernst +Date: Fri Oct 26 13:37:24 2018 -0700 + + Remove line length violations for o.e.cluster (mostly) (#34923) + + This commit removes line length violations in most of the classes under + org.elasticsearch.cluster. + +commit 5acabc47ed83eeb969c930f6cc9b298ee69c2889 +Author: Armin Braun +Date: Fri Oct 26 22:35:15 2018 +0200 + + DISCOVERY: Use Realistic Num. of Min Master Nodes (#34854) + + * DISCOVERY: Use Realistic Num. of Min Master Nodes + + * With all 3 nodes starting in parallel 2 nodes can win a master election and + start waiting for nodes to join when started in parallel. We wait for 30s on the + rest tests for the cluster health endpoint to show 3 nodes which breaks if + one of the master nodes itself waits for 30s for joining nodes, waiting for 5 seconds + fixes the issue and allows us to run with `minimum master nodes < node count` + * relates #33675 + +commit 8ce31f63435c34178ce61c05a84a9c4904259ffd +Author: Gordon Brown +Date: Fri Oct 26 13:54:45 2018 -0600 + + [ILM] Convert Explain request to new HLRC classes (#34922) + +commit d8322ca069ef21de5043a7975ccd92ba627af016 +Merge: e737ea7d4ab 5e0b524aa5e +Author: Tal Levy +Date: Fri Oct 26 12:46:21 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 9f87fdc7abf6c74571d48825b8385c68ff4dc727 +Author: Nik Everett +Date: Fri Oct 26 15:40:16 2018 -0400 + + Drop deprecationLogger from AbstractComponent (#34859) + + Drops the `deprecationLogger` from `AbstractComponent`, moving it to + places where we need it. This saves us from building a bunch of + `DeprecationLogger`s that we don't need. + + Relates to #34488 + +commit 10295b306dda364caddc08071f6ddae82b942a54 +Author: Nik Everett +Date: Fri Oct 26 15:26:14 2018 -0400 + + Core: Drop nodeName from AbstractComponent (#34487) + + `AbstractComponent` is trouble because its name implies that + *everything* should extend from it. It *is* useful, but maybe too + broadly useful. The things it offers access too, the `Settings` instance + for the entire server and a logger are nice to have around, but not + really needed *everywhere*. The `Settings` instance especially adds a + fair bit of ceremony to testing without any value. + + This removes the `nodeName` method from `AbstractComponent` so it is + more clear where we actually need the node name. + +commit e737ea7d4ab4bee9cb71d8af92abb8486f4589ed +Author: Tal Levy +Date: Fri Oct 26 12:19:52 2018 -0700 + + remove old doc placeholder and migrate ilm docs to top-level (#34615) + + we are restructuring the docs, this migrates ILM docs outside of the x-pack doc structure. + +commit 5e0b524aa5e8e7f6715b4a37288bcebcc98b6fb1 +Author: Nik Everett +Date: Fri Oct 26 14:58:50 2018 -0400 + + Contributing: Clarify width of snippets (#34910) + + Clarifies the width rules for snippets. + +commit 64a044240a6c2003268920c230f6aca202078ffd +Author: Armin Braun +Date: Fri Oct 26 20:57:57 2018 +0200 + + MINOR: Remove Deadcode in aggregtions.support (#34323) + + * Removed methods are just unused (the exceptions being isGeoPoint() and is + isFloatingPoint() but those could more efficiently be replaced by enum comparisons to simplify the code) + * Remove exceptions aren't thrown + +commit 55d48e4cc114a995c30645efd901419946c9a66f +Author: Costin Leau +Date: Fri Oct 26 21:34:01 2018 +0300 + + SQL: Handle aggregation for null group (#34916) + + When dealing with a null group, the associated metric aggs need to + return null as well + + Fix #34896 + +commit 02a342eb8c96290c46073bafb4f0e5759dad5b06 +Author: Igor Motov +Date: Fri Oct 26 14:23:10 2018 -0400 + + Tests: remove possibly unnecessary rollup job logging (#34883) + + It seems that this statement is a debug leftover since it currently + adds an error message `{"jobs":[]}` after each successful REST test. + +commit a086c665a35af32e60e6d23ab6ca75fcf085b46f +Author: Boaz Leskes +Date: Fri Oct 26 20:16:44 2018 +0200 + + HLREST: Add Clear Roles Cache API (#34187) + + Adds support for the Clear Roles Cache API to the High Level Rest + Client. As part of this a helper class, NodesResponseHeader, has been + added that enables parsing the nodes header from responses that are + node requests. + + Relates to #29827 + +commit fced5e826c011389b1734ea9dcf03080bd049ca2 +Author: Mayya Sharipova +Date: Fri Oct 26 13:56:05 2018 -0400 + + Increase the length of randomly generated field (#34857) + + * Increase the length of randomly generated field + + randomAlphaOfLength(2) produced duplicates which led to + com.fasterxml.jackson.core.JsonParseException: Duplicate field + + increase it to randomAlphaOfLength(7) + + closes #34837 + +commit aefe2909c4167c3fcc71d42ba4eeba4287e21821 +Author: Jack Conradson +Date: Fri Oct 26 09:15:35 2018 -0700 + + [Style] Remove line length violations from ingest actions (#34886) + +commit fdfdbe486d9e31530a483dcec3c77bccf7f398b4 +Author: Jason Tedor +Date: Fri Oct 26 11:23:35 2018 -0400 + + Introduce cross-cluster replication API docs (#34726) + + This commit is our first introduction to cross-cluster replication + docs. In this commit, we introduce the cross-cluster replication API + docs. We also add skelton docs for additional content that will be added + in a series of follow-up commits. + +commit a0279bc069fbf5024a2bd36b93df9fed706700f7 +Author: Jay Modi +Date: Fri Oct 26 09:21:54 2018 -0600 + + Responses can use Writeable.Reader interface (#34655) + + In order to remove Streamable from the codebase, Response objects need + to be read using the Writeable.Reader interface which this change + enables. This change enables the use of Writeable.Reader by adding the + `Action#getResponseReader` method. The default implementation simply + uses the existing `newResponse` method and the readFrom method. As + responses are migrated to the Writeable.Reader interface, Action + classes can be updated to throw an UnsupportedOperationException when + `newResponse` is called and override the `getResponseReader` method. + + Relates #34389 + +commit 5a7b8c0e697d12f7a96f72f2160e4b286266b263 +Author: Costin Leau +Date: Fri Oct 26 18:21:14 2018 +0300 + + SQL: Provide null-safe scripts for Not and Neg (#34877) + + Introduce null-safe Painless scripts for Not and Neg + Simplify script generation for Unary functions + + Close #34848 + +commit 6880e2ad79c72f025976fa42f04387047e31c2f7 +Author: Lee Hinman +Date: Fri Oct 26 09:10:55 2018 -0600 + + [TEST] Fix RolloverActionTests.mutateInstance + + In some cases it wasn't mutating and caused test failures + +commit 43f6ba1c6305a722ad90f30f5ea6ba41d6de7fb5 +Author: Jason Tedor +Date: Fri Oct 26 11:09:55 2018 -0400 + + Fix put/resume follow request parsing (#34913) + + This commit adds some fields that were missing from put follow, and + fixes a bug in resume follow. + +commit af28d1f64869d537aa3db9f3025d8a0a60d783e8 +Author: Lee Hinman +Date: Fri Oct 26 08:47:39 2018 -0600 + + Fix line length for org.elasticsearch.common.* files (#34888) + + This removes the checkstyle suppressions for things in the `common` package. + + Relates to #34884 + +commit a39a67cd38981c51c974ea6b1f3291bd7970fe9a +Author: Dimitris Athanasiou +Date: Fri Oct 26 15:34:48 2018 +0100 + + [ML] Extract common native process base class (#34856) + + We currently have two different native processes: + autodetect & normalizer. There are plans for introducing + a new process. All these share many things in common. + This commit refactors the processes to extend an + `AbstractNativeProcess` class that encapsulates those + commonalities with the purpose of reusing the code + for new processes in the future. + +commit 1b879ea8ac3a4feeb88b4be6cbe285fa16253fa4 +Author: Jim Ferenczi +Date: Fri Oct 26 16:26:45 2018 +0200 + + Refactor children aggregator into a generic ParentJoinAggregator (#34845) + + This commit adds a new ParentJoinAggregator that implements a join using global ordinals + in a way that can be reused by the `children` and the upcoming `parent` aggregation. + This new aggregator is a refactor of the existing ParentToChildrenAggregator with two main changes: + * It uses a dense bit array instead of a long array when the aggregation does not have any parent. + * It uses a single aggregator per bucket if it is nested under another aggregation. + For the latter case we use a `MultiBucketAggregatorWrapper` in the factory in order to ensure that each + instance of the aggregator handles a single bucket. This is more inlined with the strategy we use for other + aggregations like `terms` aggregation for instance since the number of buckets to handle should be low (thanks to the breadth_first strategy). + This change is also required for #34210 which adds the `parent` aggregation in the parent-join module. + + Relates #34508 + +commit 5c2c1f44c895129023bda848e718facaa6b220af +Author: Gordon Brown +Date: Fri Oct 26 08:01:38 2018 -0600 + + [Style] Fix line lengths in action.admin.indices (#34890) + + Clean up lines over 140 characters in the the + `org.elasticsearch.action.admin.indices` packages + +commit 31810837812b7b554a021d45a6cc920c7bdc1dcf +Author: markharwood +Date: Fri Oct 26 14:21:35 2018 +0100 + + HLRC - add support for source exists API (#34519) + + HLRC - add support for source exists API + API re-uses the GetRequest object (following the precedent set by the plain “exists” api). + + Relates to #27205 + +commit 306f1d78f8996439d3f966585d9b765c1a7bd300 +Author: Martijn van Groningen +Date: Fri Oct 26 15:14:24 2018 +0200 + + [CCR] Retry when no index shard stats can be found (#34852) + + Index shard stats for the follower shard are fetched, when a shard follow task is started. + This is needed in order to bootstap the shard follow task with the follower global checkpoint. + + Sometimes index shard stats are not available (e.g. during a restart) and + we fail now, while it is very likely that these stats will be available some time later. + +commit 3f1fec1813ef095d5cdc8e3c6bb2aa3b69feb090 +Author: Albert Zaharovits +Date: Fri Oct 26 15:19:35 2018 +0300 + + [Docs] audit logfile structured format (#34584) + + Documents the new structured logfile format for auditing + that was introduced by #31931. Most changes herein + are for 6.x . In 7.0 the deprecated format is gone and a + follow-up PR is in order. + +commit c42f350a81e42218bdfecd294aa0ab689b4e9029 +Author: Tanguy Leroux +Date: Fri Oct 26 12:57:31 2018 +0200 + + [Test] Fix FullClusterRestartIT.testShrink() with copy_settings param (#34853) + + The pull request #34338 added strict deprecation mode to the REST tests + and adds the copy_settings param when testing the shrink of an index. + + This parameter has been added in 6.4.0 and will be removed in 8.0, so + the test now needs to take care of the old cluster version when adding + the copy_settings param. + +commit db12005674413c752e09936eca95355e6cf75890 +Author: Armin Braun +Date: Fri Oct 26 12:56:19 2018 +0200 + + Fix LineLength Check Suppressions: index.fielddata (#34891) + + * Fix linelength suppressions in index.fielddata + * Some lines that were too long were dead code => Removed them and all code that became dead because of it + * Relates #34884 + +commit 65edec0d42cba034eca307976fd39b98ed1b1fdb +Author: Armin Braun +Date: Fri Oct 26 12:55:21 2018 +0200 + + TEST: Stablize Minio Free Port Search (#34894) + + * Binding to `0` gives us free ports that are assigned sequentially by Linux making collisions much less likely compared to manually finding a free port in a range + * Closes #32208 + +commit 33345d96efbec6c4bcc57c04e5637b89c7a0e23f +Author: David Turner +Date: Fri Oct 26 07:52:31 2018 +0100 + + Delete flaky SettingsBasedHostProviderIT test (#34813) + + testClusterFormsByScanningPorts is flaky: sometimes in CI it's not possible to + bind to any of the ports we need to in order for the port scanning to work. + This change removes this test, and #34809 describes a better way to test this + behaviour. + +commit 734088673ed84fcb6a6958bfa880cc2731deb881 +Author: David Roberts +Date: Fri Oct 26 07:45:02 2018 +0200 + + [ML] Include message in field_stats for text log files (#34861) + + This change ensures the `message` field is always included + in the `field_stats` for the semi-structured text log file + file structure. Previously it was not, as it will almost + certainly contain all distinct values. However, for + consistency in the UI it's useful to include it. + +commit be8ad674cf92c43c50720622b94e85e12b9d3683 +Author: Tim Vernum +Date: Fri Oct 26 15:28:52 2018 +1100 + + [TEST] HLRC: Expand failure messages in API checks (#34838) + + For the assertions in the "testApiNamingConventions" test that check + the API contract, this change adds details of the method that was + being checked, and the intent of the assertion (the API contract) + +commit e1fdd00420b226b33adba3675ac0ff5d6daf954c +Author: Tal Levy +Date: Thu Oct 25 21:12:19 2018 -0700 + + Lowercase static final DeprecationLogger instance names (#34887) + + After discussing on the team's FixItFriday, we concluded that + static final instance variables that are mutable should be lowercased. + + Historically, DeprecationLogger was uppercased more frequently than lowercased. + +commit 4c4afef9cc6032f4266b33ee2ddbeb2abecbc34c +Author: Tal Levy +Date: Thu Oct 25 16:13:09 2018 -0700 + + fix PolicyStatsTests: mutate kept same indicesManaged + +commit f6ce935444c6ce520f3343ef89fd312fbda835fe +Author: Tal Levy +Date: Thu Oct 25 16:00:44 2018 -0700 + + fix `GET _ilm` response with uninitialized ILM metadata (#34881) + + ILM would return a resource-not-found exception when requesting policies + while the IndexLifecycleMetaData is not initialized. The behavior here + should not be as extreme since it is not the user's fault. + + This commit changes the behavior so that it succeeds and returns no policies + when no policy names are explicitely specified, otherwise keep the same behavior + of throwing an exception + +commit f76ced4e142bc2b1124e0f9d9985f2f8e5266a80 +Author: Tal Levy +Date: Thu Oct 25 15:24:25 2018 -0700 + + update updating-policy docs (#34661) + + This contains updates to the ILM Docs explaining how updates to policies work and + affect policy execution on indices. + +commit 810cd46a30cf8b18a3518c73c263b42b96a567d7 +Merge: a4b634e60ec 3cde1356c1f +Author: Tal Levy +Date: Thu Oct 25 14:35:33 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 3cde1356c1f2c9dca23be1ded4627093c0d3477c +Author: Nik Everett +Date: Thu Oct 25 17:03:42 2018 -0400 + + XContent: Check for bad parsers (#34561) + + Adds checks for misbehaving parsers. The checks aren't perfect at all but + they are simple and fast enough that we can do them all the time so + they'll catch most badly behaving parsers. + + Closes #34351 + +commit e27a0d5b0d2cccda65731c95d39d3942acec66d1 +Author: Senthil <34865595+senthilkumarmohan@users.noreply.github.com> +Date: Thu Oct 25 21:51:18 2018 +0100 + + Docs: Align prose with snippet (#34839) + + Modified example text to include multiple index patterns used in the code + +commit 27c4d633406d1348d2b1c9f9177fdcbe28b707c1 +Author: Stéphane Campinas +Date: Thu Oct 25 22:48:06 2018 +0200 + + document the search context is freed if the scroll is not extended (#34739) + + The `fetchPhaseShouldFreeContext` returns true when there is a scroll context but the scroll parameter is null, thus freeing the search context. + + https://github.com/elastic/elasticsearch/blob/183c32d4c39948e037a7fb44dccf31ab0d60d3c3/server/src/main/java/org/elasticsearch/search/SearchService.java#L491 + +commit 59df6e868902330412107c6640dc2fdb4c511365 +Author: Nik Everett +Date: Thu Oct 25 14:02:30 2018 -0400 + + Test: Lookup node versions on rest test start (#34657) + + This is a forward port of a change made to clean up backwards + compatibility for the rollup cleanups. It makes the version of each node + available very early on in test execution. The 6.x version of the change + used those versions to control the cleanup backwards compatibility but + that isn't needed in this branch. But having the versions around *is* + useful. So this makes them available. + + Closes #34629 + +commit 06d743195091c96a45fc410759b788db60437f35 +Author: Marios Trivyzas +Date: Thu Oct 25 22:15:28 2018 +0200 + + SQL: Return error with ORDER BY on non-grouped. (#34855) + + Previously, for some queries the validation for ORDER BY + fields didn't kick in since a HAVING close or an ORDER BY + with scalar function would add `Filter` and `Project` plans + between the `OrderBy` and the `Aggregate`. + + Now the LogicalPlan tree beneath `OrderBy` is traversed and + the ORDER BY fields are properly verified. + + Fixes: #34590 + +commit cf9aff954ef916aaf1bddea362bcbb91466e60a1 +Author: Tim Brooks +Date: Thu Oct 25 13:37:49 2018 -0600 + + Reduce channels in AbstractSimpleTransportTestCase (#34863) + + This is related to #30876. The AbstractSimpleTransportTestCase initiates + many tcp connections. There are normally over 1,000 connections in + TIME_WAIT at the end of the test. This is because every test opens at + least two different transports that connect to each other with 13 + channel connection profiles. This commit modifies the default + connection profile used by this test to 6. One connection for each + type, except for REG which gets 2 connections. + +commit a4b634e60ecfb2a34f13a75bb82a35de9208a945 +Author: Lee Hinman +Date: Thu Oct 25 11:53:31 2018 -0600 + + Remove unnecessary "acknowledge: true" statements in REST tests + +commit 312df5546c2c752f4159772207e55aa1575b535d +Author: Lisa Cawley +Date: Thu Oct 25 10:32:50 2018 -0700 + + [DOCS] Updates Elasticsearch monitoring tasks (#34339) + +commit 3e7042832acfffad4de4214ad6491b147f5157b0 +Merge: 099baafe0d2 70871b5af78 +Author: Lee Hinman +Date: Thu Oct 25 11:00:36 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 099baafe0d2b4ee7cd41d9f077d97cb857948911 +Author: Gordon Brown +Date: Thu Oct 25 10:35:49 2018 -0600 + + HLRC: Split ILM request converters (#34759) + + In an effort to encapsulate different clients, the request converters + are being shuffled around. This splits the Index Lifecycle Management + request converters. + +commit 70871b5af789f6f4b499122bc302744f32588c29 +Author: Christophe Bismuth +Date: Thu Oct 25 18:12:50 2018 +0200 + + Check self references in metric agg after last doc collection (#33593) (#34001) + + * Check self references in metric agg after last doc collection (#33593) + + * Revert 0aff5a30c5dbad9f476be14f34b81e2d1991bb0f (#33593) + + * Check self refs in metric agg only once in post collection hook (#33593) + + * Remove unnecessary mocking (#33593) + +commit c0c6a28e863526da7539ec4a968ac231fa44afc0 +Author: Christoph Büscher +Date: Thu Oct 25 17:59:59 2018 +0200 + + [Docs] Add `indices.query.bool.max_clause_count` setting (#34779) + + This change adds a section about the global search setting + `indices.query.bool.max_clause_count` that limits the number of boolean clauses + allowed in a Lucene BooleanQuery. + + Closes #19858 + +commit 3225b2dcd368c4e604567d8c604491c05904216d +Author: Tanguy Leroux +Date: Thu Oct 25 17:30:25 2018 +0200 + + Add 6.6.0 version to master (#34847) + + This commit adds the 6.6.0 version constant to the master branch, and + adapts the VersionTests. + +commit d824cbe9929ea97a5fa7006c42dcc8f9cad9474a +Author: Jay Modi +Date: Thu Oct 25 08:58:21 2018 -0600 + + Test: ensure char[] doesn't being with prefix (#34816) + + The testCharsBeginsWith test has a check that a random prefix of length + 2 is not the prefix of a char[]. However, there is no check that the + char[] is not randomly generated with the same two characters as the + prefix. This change ensures that the char[] does not begin with the + prefix. + + Closes #34765 + +commit 17fe37e5ddbffd024d7b4c60cb8591c9a0668830 +Author: Tal Levy +Date: Thu Oct 25 07:30:53 2018 -0700 + + ILM Docs: update start/stop documentation (#34787) + + adds documentation explaining the different operation modes in the + context of starting and stopping ILM. + +commit 41eaa586e8c46dc871e95d70731df31cf116edea +Author: Tal Levy +Date: Thu Oct 25 07:27:04 2018 -0700 + + remove index.lifecycle.skip setting (#34823) + + With the introduction of _ilm/stop and _ilm/start APIs, the + use cases where one would only target a select group + of indices to start/stop has been reduced. Since there is no + strong use-case for skipping specific indices, it is best to + remove this functionality and only adding if later desired, with the + hopes of keeping things more simple. + +commit 21b9b024c7f57a4060d52cf7b14f65eec946bc3e +Author: Tal Levy +Date: Thu Oct 25 07:24:54 2018 -0700 + + fix PolicyStatsTests mutateInstance (#34835) + + through randomization, there is a chance that the mutateInstance + for PolicyStatsTests does not actually mutate the original object. + This PR aims to fix this + +commit 24df2eba805872eea1a159cbc1ce834575d2366c +Author: 王振荟 <605951224@qq.com> +Date: Thu Oct 25 23:22:50 2018 +0900 + + Remove static import from HLRC doc snippet (#34834) + +commit 2b652f324240fc825f23210cffb3b73ecf9fb397 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Thu Oct 25 15:52:50 2018 +0200 + + Logging: server: clean up logging (#34593) + + Replace internal deprecated calls to `Loggers.getLogger(Class)` + with direct calls to log4j `LogManager.getLogger(Class)` + +commit 185c06bb7f2bd7bff491738b01ed7716c69a70db +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Thu Oct 25 15:52:41 2018 +0200 + + Logging: tests: clean up logging (#34606) + + Replace internal deprecated calls to `Loggers.getLogger(Class)` + with direct calls to log4j `LogManager.getLogger(Class)` + +commit bd143334d3ec773d5e47090233dd6218e4e4bf98 +Author: Marios Trivyzas +Date: Thu Oct 25 14:24:11 2018 +0200 + + SQL: Fix edge case: ` IN (null)` (#34802) + + Handle the case when `null` is the only value in the list so that + it's translated to a `MatchNoDocsQuery`. + + Followup to: #34750 + +commit a69c540f12daded596ce2db931b0d3fbbe19c942 +Author: Tanguy Leroux +Date: Thu Oct 25 13:37:23 2018 +0200 + + [Test] Mute FullClusterRestartIT.testShrink() until test is fixed + + See https://github.com/elastic/elasticsearch/pull/34853 + +commit 0b26f8b14c0b453d10b446731d392f7063413ffd +Author: Colin Goodheart-Smithe +Date: Thu Oct 25 10:44:46 2018 +0100 + + Fixes NPE in multi node qa testt + +commit b97546a5a9d9ab0556fc21854a7ed7c48592c7fe +Author: Costin Leau +Date: Thu Oct 25 12:40:00 2018 +0300 + + SQL: Introduce ODBC mode, similar to JDBC (#34825) + + Close #34720 + +commit a7e08f462f103d0414635ed05cf728a4fbfc79de +Author: Andrei Stefan +Date: Thu Oct 25 12:14:49 2018 +0300 + + SQL: handle X-Pack or X-Pack SQL not being available in a more graceful way (#34736) + + Throw a different error message for a http response code of 400, but also when the error itself is of a specific type. + +commit d6dc62ef037a29c32963532a8798185ee412b747 +Author: 王振荟 <605951224@qq.com> +Date: Thu Oct 25 17:55:14 2018 +0900 + + [Docs] Add explanation for code snippets line width (#34796) + + Add "Line width for code snippets is 76 characters" to Java Language Formatting Guidelines + +commit ff49e79d405cf359e9f8d5e031117172b401d79f +Author: Nhat Nguyen +Date: Thu Oct 25 04:36:15 2018 -0400 + + CCR: Rename follow-task parameters and stats (#34836) + + * CCR: Rename follow parameters and stats + + This commit renames the follow-task parameters and its stats. + Below are the changes: + + ## Params + - remote_cluster (unchanged) + - leader_index (unchanged) + - max_read_request_operation_count -> max_read_request_operation_count + - max_batch_size -> max_read_request_size + - max_write_request_operation_count (new) + - max_write_request_size (new) + - max_concurrent_read_batches -> max_outstanding_read_requests + - max_concurrent_write_batches -> max_outstanding_write_requests + - max_write_buffer_size (unchanged) + - max_write_buffer_count (unchanged) + - max_retry_delay (unchanged) + - poll_timeout -> read_poll_timeout + + ## Stats + - remote_cluster (unchanged) + - leader_index (unchanged) + - follower_index (unchanged) + - shard_id (unchanged) + - leader_global_checkpoint (unchanged) + - leader_max_seq_no (unchanged) + - follower_global_checkpoint (unchanged) + - follower_max_seq_no (unchanged) + - last_requested_seq_no (unchanged) + - number_of_concurrent_reads -> outstanding_read_requests + - number_of_concurrent_writes -> outstanding_write_requests + - buffer_size_in_bytes -> write_buffer_size_in_bytes (new) + - number_of_queued_writes -> write_buffer_operation_count + - mapping_version -> follower_mapping_version + - total_fetch_time_millis -> total_read_time_millis + - total_fetch_remote_time_millis -> total_read_remote_exec_time_millis + - number_of_successful_fetches -> successful_read_requests + - number_of_failed_fetches -> failed_read_requests + - operation_received -> operations_read + - total_transferred_bytes -> bytes_read + - total_index_time_millis -> total_write_time_millis [?] + - number_of_successful_bulk_operations -> successful_write_requests + - number_of_failed_bulk_operations -> failed_write_requests + - number_of_operations_indexed -> operations_written + - fetch_exception -> read_exceptions + - time_since_last_read_millis -> time_since_last_read_millis + + * add test for max_write_request_(operation_count|size) + +commit 13ca942152c611b2c44689e226cdfdf00b59e71a +Author: Alpar Torok +Date: Thu Oct 25 10:03:23 2018 +0300 + + Switch build-tools to latest target version (#34746) + + - we already require Java 11 to build, yet we target the minimum + supported version in build-tools ( currently 8 ) + - this is because we have some checks that are executed in a new JVM + which could be running the minimum version. + - For everything else it would be nice to be able to use new features, + like the new process API. + + With this change, we selectively compile the few classes that need an + older target version and move everything over to Java 10. + Unfortunately the current Gradle version does not support 11 as a target + version yet. + +commit bb807b147b5a51e108621c51eb4a67a0fc3e98ec +Author: Martijn van Groningen +Date: Thu Oct 25 08:09:45 2018 +0200 + + fixed test + +commit e7ced94a65886eade5d6bbb461e194e493094228 +Author: Armin Braun +Date: Thu Oct 25 07:51:14 2018 +0200 + + NETWORKING: Add SSL Handler before other Handlers (#34636) + + * NETWORKING: Add SSL Handler before other Handlers + + * The only way to run into the issue in #33998 is for `Netty4MessageChannelHandler` + to be in the pipeline while the SslHandler is not. Adding the SslHandler before any + other handlers should ensure correct ordering here even when we handle upstream events + in our own thread pool + * Ensure that channels that were closed concurrently don't trip the assertion + * Closes #33998 + +commit 59536966c2c90256aee87101c70b8074b532b165 +Author: Alpar Torok +Date: Thu Oct 25 08:50:50 2018 +0300 + + Add a new "contains" feature (#34738) + + The contains syntax was added in #30874 but the skips were not properly + put in place. + The java runner has the feature so the tests will run as part of the + build, but language clients will be able to support it at their own + pace. + +commit 153157e56d5f8509b5cc503de56113f7025d4c4f +Author: Jason Tedor +Date: Wed Oct 24 23:17:45 2018 -0400 + + Enable ingest attachment docs tests on JDK 11 (#34770) + + These were disabled because ingest attachement was not playing well with + JDK 11. We have addressed that by upgrading the Tika dependency, yet + forgot to reenable these. This commit enables these tests again. + +commit 687dc1eb11265fb0abc6a0bce8075744dd69efa8 +Author: Ryan Ernst +Date: Wed Oct 24 15:03:38 2018 -0700 + + Scripting: Remove SearchScript (#34730) + + This commit removes the last non context based script class. + +commit d7afd7c123babeb1481aacdb735743af719ac866 +Author: Mayya Sharipova +Date: Wed Oct 24 17:57:35 2018 -0400 + + Reduce the number of callouts to 15 in docs + + relates to #33447 + +commit 6fe0e62b7a29242cd19ec54bac0364e0e04e4be5 +Author: Martijn van Groningen +Date: Wed Oct 24 23:48:49 2018 +0200 + + [CCR] Added write buffer size limit (#34797) + + This limit is based on the size in bytes of the operations in the write buffer. If this limit is exceeded then no more read operations will be coordinated until the size in bytes of the write buffer has dropped below the configured write buffer size limit. + + Renamed existing `max_write_buffer_size` to ``max_write_buffer_count` to indicate that limit is count based. + + Closes #34705 + +commit d94406a68af6d0d293fed7845c1efcfd49d84516 +Author: Mark Tozzi +Date: Wed Oct 24 17:01:22 2018 -0400 + + HLRC: Deactivate Watch API (#34192) + + Relates to #29827 + +commit 3211760e2cd5909217c9acc150bc676f842990d7 +Author: Mayya Sharipova +Date: Wed Oct 24 16:35:20 2018 -0400 + + Correct term-vectors.asciidoc + + relates to #33447 + +commit 6cceba9b54083cc3c3f0efa7327a2c5dcabf7b11 +Author: Mayya Sharipova +Date: Wed Oct 24 15:59:58 2018 -0400 + + Correct CRUDDocumentationIT.java + + relates to #33447 + +commit 1b085252c31a09b24759773ba8069d93de5753ba +Author: Jack Conradson +Date: Wed Oct 24 12:57:28 2018 -0700 + + [Painless] Add instance bindings (#34410) + + This change adds instance bindings to Painless. This binding allows a whitelisted + method to be called on an instance instantiated prior to script compilation. + Whitelisting must be done in code as there is no practical way to instantiate a + useful instance from a text file (see the tests for an example). Since an + instance can be shared by multiple scripts, each method called must be + thread-safe. + +commit 5f588180f942b07bcf462e4416d3ee5f04c8b81f +Author: Andrey Atapin +Date: Thu Oct 25 00:53:31 2018 +0500 + + Improve IndexNotFoundException's default error message (#34649) + + This commit adds the index name to the error message when an index is not found. + +commit 04f3e67c7792d4ecd46ee42e3428f4941143ca72 +Author: Stéphane Campinas +Date: Wed Oct 24 21:31:29 2018 +0200 + + Remove redundant method from RestClearScrollAction (#34268) + + The check for null argument is already done in `splitStringByCommaToArray`, hence it can be removed, which allows us to remove the whole splitScrollIds private method. + +commit d51bc05dce6432a071b7c63de8efaeb9ace94115 +Author: Luca Cavanna +Date: Wed Oct 24 21:27:07 2018 +0200 + + [TEST] Improve validation of do sections (#34734) + + We throw parsing exception when an unknown array is found, but we don't when an unknown top-level field is found. This commit makes sure that unsupported top-level fields are not ignored in a do section. + + Closes #34651 + +commit cd27b0b996d5cc81d16f57623f1e7cc5eba35c6f +Author: Benjamin Trent +Date: Wed Oct 24 14:25:10 2018 -0500 + + Revert "Rollup add default metrics to histo groups (#34534)" (#34815) + + This reverts commit 4236358f5d53ab4f0d8457faacb19983b6b1ab82. + +commit 7570d692544ea262ead23455c9b3640d5ddcc820 +Author: Costin Leau +Date: Wed Oct 24 22:03:11 2018 +0300 + + SQL: Introduce support for IP fields (#34758) + + IP fields are recognized and can be used through-out the query + + Close #32499 + +commit 98cd7ca861e4397f9bb715987bd928d74ec18c85 +Author: Marios Trivyzas +Date: Wed Oct 24 20:36:30 2018 +0200 + + SQL: Fix queries with filter resulting in NO_MATCH (#34812) + + Previously, `Mapper` was returning an incorrect plan which resulted in an + ``` + SQLFeatureNotSupportedException: Found 1 problem(s) + line 1:8: Unexecutable item + ``` + + Queries with a `WHERE` and/or `HAVING` clause which results in NO_MATCH + are now handled correctly and return 0 rows. + + Fixes: #34613 + + Co-authored-by: Costin Leau + +commit bf4d90a5dc2fe2ee51ec182a166e636d88bf5d05 +Author: Mayya Sharipova +Date: Wed Oct 24 14:27:22 2018 -0400 + + HLRC API for _termvectors (#33447) + + * HLRC API for _termvectors + + relates to #27205 + +commit e7fddb5c933e542ab4aebd7196cb7a97fab74b57 +Author: Colin Goodheart-Smithe +Date: Wed Oct 24 18:28:46 2018 +0100 + + Adds usage data for ILM (#33377) + + * Adds usage data for ILM + + * Adds tests for IndexLifecycleFeatureSetUsage and friends + + * Adds tests for IndexLifecycleFeatureSet + + * Fixes merge errors + + * Add number of indices managed to usage stats + + Also adds more tests + + * Addresses Review comments + +commit f19565c3e035f6aab5a07f7c8b81bee86d72fca8 +Author: Costin Leau +Date: Wed Oct 24 19:31:26 2018 +0300 + + SQL: Verifier allows aliases aggregates for sorting (#34773) + + Improve Verifier rule that prevented grouping with aliases inside aggregates + to not be accepted for ordering. + + Close #34607 + +commit f7a6fb288f4e71f6946c6fc0ce1c588c68951286 +Author: markharwood +Date: Wed Oct 24 16:58:31 2018 +0100 + + Fix compilation error peculiar to eclipse2018-09 (#34798) + + Fix compilation error peculiar to eclipse2018-09. + Added generics to make eclipse compiler happy + +commit d73768f812a305fd7ce71163994452e2629bb184 +Author: Nhat Nguyen +Date: Wed Oct 24 11:19:39 2018 -0400 + + CCR: Do not follow if leader does not have soft-deletes (#34767) + + We should not create a follower index and abort a follow request if the + leader does not have soft-deletes. Moreover, we also should not + auto-follow an index if it does not have soft-deletes. + +commit 0a85997cb00708c8b0ab5da22e3742a4b4ee07d1 +Author: Armin Braun +Date: Wed Oct 24 17:18:52 2018 +0200 + + TESTS: Use File Based Discovery in REST Tests (#34560) + + * For `6.5+` use file based discovery in REST tests + * Relates #33675 + +commit e0d7808148c6904dfdc66622587052a92d6a4f0d +Author: Nik Everett +Date: Wed Oct 24 11:15:44 2018 -0400 + + REST: No strict warning testing for head body test + + Our tests that HEAD requests don't have a body was hitting a warning. + For now we'll run that test without "strict" warnings mode enabled. + +commit 4370b44ef000c9038c3b97fe0d1d26e20c0ca55b +Author: Alpar Torok +Date: Wed Oct 24 18:15:25 2018 +0300 + + Rename testcluster extension to align with plugin name (#34751) + +commit c7fe87e43f800f8d399865943da4a7e5aacbb835 +Author: Colin Goodheart-Smithe +Date: Wed Oct 24 16:14:59 2018 +0100 + + Removes Set Policy API in favour of setting index.lifecycle.name directly (#34304) + + * Removes Set Policy API in favour of setting index.lifecycle.name + directly + + * Reinstates matcher that will still be used + + * Cleans up code after rebase + + * Adds test to check changing policy with ndex settings works + + * Fixes TimeseriesLifecycleActionsIT after API removal + + * Fixes docs tests + + * Fixes case on close where lifecycle service was never created + +commit 0f1a5ecf6118e939a472331537f96d55f6fddda3 +Author: Shaunak Kashyap +Date: Wed Oct 24 07:20:46 2018 -0700 + + Adding stack_monitoring_agent role (#34369) + + * Adding stack_monitoring_agent role + + * Fixing checkstyle issues + + * Adding tests for new role + + * Tighten up privileges around index templates + + * s/stack_monitoring_user/remote_monitoring_collector/ + remote_monitoring_user + + * Fixing checkstyle violation + + * Fix test + + * Removing unused field + + * Adding missed code + + * Fixing data type + + * Update Integration Test for new builtin user + +commit 795d57b4f91f9b7d90a8a322e282dcbe13f866f3 +Author: Alpar Torok +Date: Wed Oct 24 16:05:50 2018 +0300 + + Auto configure all test tasks (#34666) + + With this change, we apply the common test config automatically to all + newly created tasks instead of opting in specifically. + + For plugin authors using the plugin externally this means that the + configuration will be applied to their RandomizedTestingTasks as well. + + The purpose of the task is to simplify setup and make it easier to + change projects that use the `test` task but actually run integration + tests to use a task called `integTest` for clarity, but also because + we may want to configure and run them differently. + E.x. using different levels of concurrency. + +commit 4c73854da78d9f95e1c9a9aa7ae973e870102ba6 +Author: Marios Trivyzas +Date: Wed Oct 24 14:42:40 2018 +0200 + + SQL: Implement null handling for `IN(v1, v2, ...)` (#34750) + + Implemented null handling for both the value tested but also for + values inside the list of values tested against. + + The null handling is implemented for local processors, painless scripts + and Lucene Terms queries making it available for `IN` expressions occuring + in `SELECT`, `WHERE` and `HAVING` clauses. + + Closes: #34582 + +commit d5ad3de42ee7f1b45d57922e89e6d7a8b8f6e51e +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Wed Oct 24 14:21:24 2018 +0200 + + [test] Introduce strict deprecation mode for REST tests (#34338) + + #33708 introduced a strict deprecation mode that makes a REST request + fail if there is a warning header in the response returned by + Elasticsearch (usually a deprecation message signaling that a feature + or a field has been deprecated). + + This change adds the strict deprecation mode into the REST integration + tests, and makes the tests fail if a deprecated feature is used. Also + any test using a deprecated feature has been modified to pass the build. + + The YAML integration tests already analyzed HTTP warnings so they do + not use this mode, keeping their "expected vs actual" behavior. + +commit 89404ced9b1f955eed2c1f87015baf8b7108e561 +Author: Colin Goodheart-Smithe +Date: Wed Oct 24 13:05:41 2018 +0100 + + Fixes SOTP docs test to restart ILM after + +commit 76240e6bbe5dd2e53db4550152b7f3b6fa54903a +Author: Martijn van Groningen +Date: Wed Oct 24 13:39:36 2018 +0200 + + [CCR] Renamed leader_cluster to remote_cluster (#34776) + + and also some occurrences of clusterAlias to remoteCluster. + + Closes #34682 + +commit be907516adb1d4409af349a7b84c8ea4f4ac8b2b +Author: Boaz Leskes +Date: Wed Oct 24 13:32:48 2018 +0200 + + Change ShardFollowTask defaults (#34793) + + Per #31717 this commit changes the defaults to the following: + + Batch size of 5120 ops. + Maximum of 12 concurrent read requests. + Maximum of 9 concurrent write requests. + + This is not necessarily our final values but it's good to have these as defaults for the purposes of initial testing. + +commit fe54f73434af0bda3b42bfa837b56e916c01251f +Author: hanbing0715 +Date: Wed Oct 24 18:55:21 2018 +0800 + + Persist generated public keys unencrypted (#34626) + + The changes introduced in cca1a2a mean that we should + not encrypt the public keys that might be generated by + the key-pair-generator when storing the file, as the code + that would consume them assumes that they are not encrypted + +commit 7a3cd10718d775416913b2a8187046e7b476e0be (es/zen2) +Author: Andrey Ershov +Date: Wed Oct 24 13:45:12 2018 +0300 + + [Zen2] Change MetaDataStateFormat write semantics (#34709) + + Currently, if MetaDataStateFormat.write throws an IOExceptions if there was some problem with persisting state to disk. If an exception is thrown, loadLatestState may read either old state or new state. This is not enough for the Zen2 algorithm. In case of failure, we need to distinguish between 2 cases: storage is left in clean state or storage is left in a dirty state. + If storage is left in the clean state, loadLatestState may read only old state. If storage is left in a dirty state, loadLatestState may read either old or new state. + If an exception occurs when writing the manifest file to disk this distinction is important for Zen2. If storage is clean, the node can continue to be a part of the cluster and may try to accept further cluster state updates (if it fails to accept cluster state updates it will be kicked off from the cluster using different mechanism). But if storage is dirty, the node should be restarted and it will be able to start up successfully only once it successfully re-writes manifest file to disk. + This commit changes MetaDataStateFormat.write signature, replacing IOException with WriteStateException, which “isDirty” method could be used to distinguish between 2 failure cases. + We need to minimise the number of failures, that leave storage in a dirty state. That’s why this PR changes the algorithm that is used to store state to disk. It has the following layout: + + 1. For the first state location, create and fsync tmp file with state content. + 2. For each extra location, copy and fsync tmp file with state content. + 2. Atomically rename tmp file in the first location. + 3. For each extra location, atomically rename tmp file. + 4. For each location, fsync state directory. + 5. Perform cleanup of old files, ignoring exceptions. + If an exception occurs in steps 1-3, storage is clearly in the clean state. If an exception occurs in step 5, storage is clearly in dirty state. Exception in step 4 is questionable, there are 2 options: + 1. Consider it as a failure. If the first disk fails, state disappears. So this is a failure and storage is in a dirty state. + 2. Do not consider it as failure at all, ignore disk failures. + This commit prefers 1st approach and MetaDataTestFormatTests.testFailRandomlyAndReadAnyState tests for disk failures. + +commit 18e004c3eaa18178d5523c06a91bc4d29a4a7816 +Author: Martijn van Groningen +Date: Wed Oct 24 09:27:08 2018 +0200 + + [CCR] Removed unused static parse field. + +commit 18007a29b2eaa499fcfa7aa1c82b0c09400a1b28 +Author: Martijn van Groningen +Date: Wed Oct 24 08:38:25 2018 +0200 + + [CCR] Made leader cluster required in shard follow task. + Left over from #34580 + +commit abf8cb670621cb8b457409b301911c955b654b7f +Author: Martijn van Groningen +Date: Wed Oct 24 08:12:39 2018 +0200 + + [CCR] Cleanup pause follow action (#34183) + + * Change the `TransportPauseFollowAction` to extend from `TransportMasterNodeAction` + instead of `HandledAction`, this removes a sync cluster state api call. + * Introduced `ResponseHandler` that removes duplicated code in `TransportPauseFollowAction` and + `TransportResumeFollowAction`. + * Changed `PauseFollowAction.Request` to not use `readFrom()`. + +commit 52266d8b115ce8633e57ff1e5abfbb951496e67b +Author: Nhat Nguyen +Date: Tue Oct 23 21:08:34 2018 -0400 + + TEST: Clone replicas list when compute replication targets (#34728) + + In #34407, we supposed to clone the list of replicas of ReplicationGroup + when computing replication targets, but somehow we missed it. If we + don't clone the list, a WriteReplicationAction may use an old + ReplicationTargets which consists replicas which are removed from the + current list of replicas + + Relates #34407 + Closes #33457 + +commit 8da1c9626a19853a692c30bbd30428e0a38c5f75 +Author: Ryan Ernst +Date: Tue Oct 23 18:07:53 2018 -0700 + + Scripting: Add back params._source access in scripted metric aggs (#34777) + + Access to special variables _source and _fields were accidentally + removed in recent refactorings. This commit adds them back, along with a + test. + + closes #33884 + +commit 596b5cf108083bb898c386cb620c8bbc2d87b742 +Author: Ryan Ernst +Date: Tue Oct 23 17:26:17 2018 -0700 + + Test: Fix last reference to SearchScript (#34731) + + This was accidentally left over when converting to FieldScript. + + closes #34683 + +commit da20dfd81c46c61302d32c7d4eb9a56cf24e33fc +Author: Gordon Brown +Date: Tue Oct 23 16:35:10 2018 -0600 + + Add cluster-wide shard limit warnings (#34021) + + In a future major version, we will be introducing a soft limit on the + number of shards in a cluster based on the number of nodes in the + cluster. This limit will be configurable, and checked on operations + which create or open shards and issue a warning if the operation would + take the cluster over the limit. + + There is an option to enable strict enforcement of the limit, which + turns the warnings into errors. In a future release, the option will be + removed and strict enforcement will be the default (and only) behavior. + +commit c5a07393810990234555ad00b48238b7f4a8846a +Author: Julie Tibshirani +Date: Tue Oct 23 15:26:39 2018 -0700 + + Mute SettingsBasedHostProviderIT to avoid future test flakes. + +commit 7580f59c9fa87a6361ad4e7182cbaed641ecf770 +Author: Julie Tibshirani +Date: Tue Oct 23 14:27:47 2018 -0700 + + Mute FullClusterRestartIT#testSqlFailsOnIndexWithTwoTypes while we await a fix. + +commit 17adfeb20fd6ca31ddfda09ec61a07b5bfd3f24c +Author: Gordon Brown +Date: Tue Oct 23 15:59:11 2018 -0600 + + HLRC: Standardize access in *RequestConverters (#34768) + + With the move to separate RequestConverters classes for each client, + some of the access restrictions on the new classes are more open than + the prior RequestConverters classes. This standardizes the + *RequestConverters classes as package-private, final, and with a private + constructor so that no instances of the can be inadvertently created. + +commit 0efba0675e633e98660447dadd0b71183226934e +Author: Martijn van Groningen +Date: Tue Oct 23 23:24:32 2018 +0200 + + [CCR] Add qa test library (#34611) + + * Introduced test qa lib that all CCR qa modules depend on to avoid + test code duplication. + +commit c5a264e77f2bc577b2ede77b1c1156c2aa33e66d +Author: Lee Hinman +Date: Tue Oct 23 15:20:41 2018 -0600 + + Ensure phase_time is set when in the "new" phase (#34280) + + Since there's no transition into the "new" phase it wasn't set until the "hot" + phase, so now we initialize it when initializing the policy context. + + Resolves #34277 + +commit 1f42024243aa083d8b1cdb6827d7a691fae6d071 +Author: Julie Tibshirani +Date: Tue Oct 23 14:00:53 2018 -0700 + + Mute NodeSubclassTests#testReplaceChildren while we await a fix. + +commit f79bdec58af007c247069f64b8f828ee374a7f24 +Author: Armin Braun +Date: Tue Oct 23 22:36:57 2018 +0200 + + INGEST: Document Pipeline Processor (#33418) + + * Added documentation for Pipeline Processor + * Relates #33188 + +commit 299d044bfceaf5391cda1bc19f92f8bc2cbf87ae +Author: Zachary Tong +Date: Tue Oct 23 16:01:01 2018 -0400 + + Collapse pipeline aggs into single package (#34658) + + - Restrict visibility of Aggregators and Factories + - Move PipelineAggregatorBuilders up a level so it is consistent with + AggregatorBuilders + - Checkstyle line length fixes for a few classes + - Minor odds/ends (swapping to method references, formatting, etc) + +commit 067d135bfdffc8e442a7b6f34ca5e20b09b4d67a +Author: Jake Landis +Date: Tue Oct 23 13:49:25 2018 -0500 + + ingest: document index.default_pipeline (#34500) + +commit 5a4866f67d3aac0580cc3b1bc692af8e1b38a3be +Author: Julie Tibshirani +Date: Tue Oct 23 11:36:24 2018 -0700 + + Mute CharArraysTests#testCharsBeginsWith while we await a fix. + +commit a8e1ee34ca040212a2677c2d46cdc5b8fb32dcad +Author: Jake Landis +Date: Tue Oct 23 13:28:44 2018 -0500 + + ingest: document fields that support templating (#34536) + + This change also updates many of the examples to use ecs as the example. + Some additional minor improvements are also included. + + Part of #33188 + +commit e242fd2e42fb6759b2bf4c0a73451e814b1a0e20 +Author: Nhat Nguyen +Date: Tue Oct 23 14:23:29 2018 -0400 + + CCR: Add TransportService closed to retryable errors (#34722) + + Both testFollowIndexAndCloseNode and testFailOverOnFollower failed + because they responded to the FollowTask a TransportService closed + exception which is currently considered as a fatal error. This behavior + is not desirable since a closing node can throw that exception, and we + should retry in that case. + + This change adds TransportService closed error to the list of retryable + errors. + + Closes #34694 + +commit 90fd15bb563160e59b582f359e764aa99a966bc6 +Author: Julie Tibshirani +Date: Tue Oct 23 11:14:31 2018 -0700 + + Mute RollupIndexerIndexingTests#testRandomizedDateHisto as we await a fix. + +commit ed817fb2659d4a4adbe10601886cfcbacd153c7d +Author: Martijn van Groningen +Date: Tue Oct 23 19:37:45 2018 +0200 + + [CCR] Move leader_index and leader_cluster parameters from resume follow to put follow api (#34638) + + As part of this change the leader index name and leader cluster name are + stored in the CCR metadata in the follow index. The resume follow api + will read that when a resume follow request is executed. + +commit c447fc258a2fc8e7c193c928310fb4e49844e4a2 +Author: Jake Landis +Date: Tue Oct 23 12:30:23 2018 -0500 + + ingest: documentation for the drop processor (#34570) + +commit eaa05fe76236fdba70e3c54c3b830f2269e9669a +Author: Emily S +Date: Tue Oct 23 19:09:09 2018 +0200 + + [TEST] Minor updates to rest api spec tests (#34551) + +commit 62ac2fa5ec57a4701323d1c2a0077e2179a7d422 +Merge: 59b28da2517 89dc07bdd95 +Author: Tal Levy +Date: Tue Oct 23 09:43:46 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 89dc07bdd958574810fea533c87d2bf3c3d2b38a +Author: Jake Landis +Date: Tue Oct 23 11:33:48 2018 -0500 + + ingest: better support for conditionals with simulate?verbose (#34155) + + This commit introduces two corrections to the way simulate?verbose + handles conditionals on processors. + + 1) Prior to this change when executing simulate?verbose for + processors with conditionals that evaluate to false, that processor + would still be displayed in the result set. What was displayed was + correct, such that no changes to the document occurred. However, if the + conditional evaluates to false, the processor should not even be + displayed. + + 2) Prior to this change when executing simulate?verbose for + pipeline processors with conditionals, the individual steps would no + longer be displayed. Commit e37e5df addressed the issue, but + failed account for a conditional on the pipeline processor. Since + a pipeline processor can introduce cycles and is effectively a + single processor that encapsulates multiple other processors that + are potentially guarded by a single conditional, special handling is + needed to for pipeline and conditional pipeline processors. + +commit 4dbf498721a7f7b9d5d1274859ed0752c5b30934 +Author: Zachary Tong +Date: Tue Oct 23 12:23:22 2018 -0400 + + [Rollup] Job deletion should be invoked on the allocated task (#34574) + + We should delete a job by directly talking to the allocated + task and telling it to shutdown. Today we shut down a job + via the persistent task framework. This is not ideal because, + while the job has been removed from the persistent task + CS, the allocated task continues to live until it gets the + shutdown message. + + This means a user can delete a job, immediately delete + the rollup index, and then see new documents appear in + the just-deleted index. This happens because the indexer + in the allocated task is still running and indexes a few + more documents before getting the shutdown command. + + In this PR, the transport action is changed to a TransportTasksAction, + and we invoke onCancelled() directly on the matching job. + The race condition still exists after this PR (albeit less likely), + but this was a precursor to fixing the issue and a self-contained + chunk of code. A second PR will followup to fix the race itself. + +commit 46b49b01c41ed3c984db548cb45cde86ebdc21ca +Author: Albert Zaharovits +Date: Tue Oct 23 18:51:59 2018 +0300 + + [DOCS] .Security index is never auto created (#34589) + +commit 5923ea536e01c84f2f43fa268670c6e371548462 +Author: Nhat Nguyen +Date: Tue Oct 23 11:51:17 2018 -0400 + + CCR: Requires soft-deletes on the follower (#34725) + + Since #34412 and #34474, a follower must have soft-deletes enabled + to work correctly. This change requires soft-deletes on the follower. + + Relates #34412 + Relates #34474 + +commit cadd6731e78d9b5fd8f79b76b18bb3ad9d19687d +Author: Jake Landis +Date: Tue Oct 23 10:50:49 2018 -0500 + + re-enable bwc tests (#34743) + +commit 11881e7b5036c7faa7362d5574e18637052c7964 +Author: Albert Zaharovits +Date: Tue Oct 23 18:50:20 2018 +0300 + + Empty GetAliases authorization fix (#34444) + + This fixes a bug about aliases authorization. + That is, a user might see aliases which he is not authorized to see. + This manifests when the user is not authorized to see any aliases + and the `GetAlias` request is empty which normally is a marking + that all aliases are requested. In this case, no aliases should be + returned, but due to this bug, all aliases will have been returned. + +commit f0f732908ee3559ac08abfde3308cec5bf444b54 +Author: Armin Braun +Date: Tue Oct 23 17:37:30 2018 +0200 + + INGEST: Document Processor Conditional (#33388) + + * INGEST: Document Processor Conditional + + Relates #33188 + +commit 59b28da25173e12929eded5e2267f46f2e021433 +Author: Gordon Brown +Date: Tue Oct 23 08:46:27 2018 -0600 + + [ILM] Add null check to CopyExecutionStateStep (#34619) + + Adds a null check for IndexMetaData similar to those in #33455. + +commit e6d87cc09f5b22783c991f31ddc0e765e82141ca +Author: Martijn van Groningen +Date: Tue Oct 23 16:41:06 2018 +0200 + + [CCR] Add total fetch time leader stat (#34577) + + Add total fetch time leader stat, that + keeps track how much time was spent on fetches + from the leader cluster perspective. + +commit ca6808e55de5b9b5d0daa074c1512366fcb57fde +Author: Costin Leau +Date: Tue Oct 23 17:07:51 2018 +0300 + + SQL: Support pattern against compatible indices (#34718) + + Extend querying support on multiple indices from being strictly + identical to being just compatible. + Use FieldCapabilities API (extended through #33803) for mapping merging. + + Close #31837 #31611 + +commit 36baf3823da394068e4380955c588008b30e83da +Author: Martijn van Groningen +Date: Tue Oct 23 15:48:51 2018 +0200 + + [CCR] Auto follow pattern APIs adjustments (#34518) + + * Changed the resource id of auto follow patterns to be a user defined name + instead of being the leader cluster alias name. + * Fail when an unfollowed leader index matches with two or more auto follow patterns. + +commit 583f2852f0ce4d11a809e921c12f7ddaf58574e7 +Author: Christoph Büscher +Date: Tue Oct 23 15:44:47 2018 +0200 + + [Test] Remove dead code from ExceptionSerializationTests (#34713) + + The `ignore` set contains entries of type Class, but the check is performed + on Path objects. This always returns false so is useless currently. Looking at + the first commit of this test that already shows this behaviour this never + excluded anything, so it can be removed. + +commit f91cc13c1849f2a003bbf039c2dc8609f4418b8c +Author: Przemyslaw Gomulka +Date: Tue Oct 23 15:28:25 2018 +0200 + + A small typo in migration-assistance doc (#34704) + + typo - missing word "be" + +commit ad94e793504a7e4ddc88eedc888b0abdf580f90a +Author: Jake Landis +Date: Tue Oct 23 07:30:52 2018 -0500 + + ingest: processor stats (#34724) + + This change introduces stats per processors. Total, time, failed, + current are currently supported. All pipelines will now show all + top level processors that belong to it. Failure processors are not + displayed, however, the time taken to execute the failure chain is part + of the stats for the top level processor. + + The processor name is the type of the processor, ordered as defined in + the pipeline. If a tag for the processor is found, then the tag is + appended to the type. + + Pipeline processors will have the pipeline name appended to the name of + the name of the processors (before the tag if one exists). If more + then one pipeline is used to process the document, then each pipeline + will carry its own stats. The outer most pipeline will also include the + inner most pipeline stats. + + Conditional processors will only included in the stats if the condition evaluates + to true. + +commit 4a8386f271f449d01d5f81d891d2b3d17272f1e0 +Author: Marios Trivyzas +Date: Tue Oct 23 14:28:23 2018 +0200 + + SQL: Implement IN(value1, value2, ...) expression. (#34581) + + Implement the functionality to translate the + `field IN (value1, value2,...)` expressions to proper Lucene queries + or painless script or local processors depending on the use case. + + The `IN` expression can be used in SELECT, WHERE and HAVING clauses. + + Closes: #32955 + +commit 123f784e3249919c23141c6a943d14a9da586337 +Author: Igor Motov +Date: Tue Oct 23 07:55:41 2018 -0400 + + Tests: Add checks to GeoDistanceQueryBuilderTests (#34273) + + Adds checks for parsed geo distance query. It is a bit hack-ish since it + compares with query's toString() output, but it is better than no + checks. The parsed query itself has default visibility, so we cannot + access it here unless we move the test to org.apache.lucene.document + package. + + Fixes #34043 + +commit 8e155b8430d0d1d216aabb10c680f7fbd1318871 +Author: Armin Braun +Date: Tue Oct 23 13:43:26 2018 +0200 + + INGEST: Rename Pipeline Processor Param. (#34733) + + * `name` is more readable/ergnomic than having `pipeline` twice + +commit 83fd93b2fd654920f0ff5d3d80f3d48b40182c88 +Author: Alexander Reelsen +Date: Tue Oct 23 13:26:02 2018 +0200 + + Core: Move IndexNameExpressionResolver to java time (#34507) + + This switches from joda time to java time when resolving index names + using date math. This commit also removes two non registered settings + from the code, which could not be used anyway. An unused method was + removed as well. + + Relates #27330 + +commit 183c32d4c39948e037a7fb44dccf31ab0d60d3c3 +Author: Frederik Bosch +Date: Tue Oct 23 12:25:04 2018 +0200 + + [DOCS] Force Merge: clarify execution and storage requirements (#33882) + +commit 4bda9bdab0bebcf4a85002c17e27d0713eccc84e +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Tue Oct 23 11:36:42 2018 +0200 + + TESTING.asciidoc fix examples using forbidden annotation (#34515) + + Clean up examples not to use forbidden test annotation `@Nightly`. + remove references to unused annotations + +commit e9e140790a5279f0a00f75c036acf9ab6ae91eb7 +Author: Marios Trivyzas +Date: Tue Oct 23 11:21:15 2018 +0200 + + SQL: Implement `CONVERT`, an alternative to `CAST` (#34660) + + `CONVERT` works exactly like cast with slightly different syntax: + `CONVERT(, AS )` + + Moreover it support format of the MS-SQL data types `SQL_`, + e.g.: `SQL_INTEGER` + + Closes: #34513 + +commit 5fb4f32c9101a0ac9ec4e6e40c693501f75c3aca +Author: Alpar Torok +Date: Tue Oct 23 12:13:49 2018 +0300 + + [Build] Simplify testclusters configuration (#34334) + + Remove interface, prefer straught implementaton instead. + +commit 1eb76f16b1ec9a635e79d135c8a21e39126db765 +Author: Marios Trivyzas +Date: Tue Oct 23 11:11:22 2018 +0200 + + SQL: Allow min/max aggregates on date fields (#34699) + + Allow `MIN()` and `MAX()` aggregate functions to operate + also on arguments of type DATE apart from the numeric ones. + + Fixes: #34477 + +commit 0536635c44b56687f6aeea241b758e2c37363d18 +Author: Alpar Torok +Date: Tue Oct 23 12:06:46 2018 +0300 + + Upgrade forbiddenapis to 2.6 (#33809) + + * Upgrade forbiddenapis to 2.6 + + Closes #33759 + + * Switch forbiddenApis back to official plugin + + * Remove CLI based task + + * Fix forbiddenApisJava9 + +commit 91434f7721954465e21c52373dcb53f5881d6160 +Author: Andrei Stefan +Date: Tue Oct 23 11:04:16 2018 +0300 + + SQL: the SSL default configuration shouldn't override the https protocol if used (#34635) + + * The default SSL option shouldn't override the https protocol if specified. Fixes https://github.com/elastic/elasticsearch/issues/33817 + +commit 9b80017f9501e767e8b4443a3daf01674787df01 +Merge: 67bfdb16ad0 f55843c9f0a +Author: Tal Levy +Date: Mon Oct 22 21:03:13 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit f55843c9f0a9431817240d5955e1b31acf50c39e +Author: Jason Tedor +Date: Mon Oct 22 19:41:12 2018 -0400 + + Enable docs tests to have tear down snippet (#34716) + + This commit adds the ability for docs tests to add a tear down + snippet. This snippet will be converted to a tear down section of the + generated REST tests. + +commit d4bb3d1ce5e13d73a53e6a379668f2cb246653bd +Author: Tim Brooks +Date: Mon Oct 22 19:01:28 2018 -0400 + + Update SSLDriver for JDK 11 changes (#34398) + + JDK11 introduced some changes with the SSLEngine. A number of error + messages were changed. Additionally, there were some behavior changes + in regard to how the SSLEngine handles closes during the handshake + process. This commit updates our tests and SSLDriver to support these + changes. + +commit 67bfdb16ad0a1039686d3a74b1705c613e96e231 +Merge: 910e97422fa ae1e46b852b +Author: Tal Levy +Date: Mon Oct 22 13:09:37 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit ae1e46b852b054cf9afa3d73006a81907a9c04d0 (es/Swap_Leadfoot_Out_From_Under_The_Hood) +Author: jaymode +Date: Mon Oct 22 14:01:44 2018 -0600 + + Test: add empty test to PainlessDomainSplitIT + + All of the tests in PainlessDomainSplitIT have an awaitsfix, which + causes the build to fail since no tests are run. This adds an empty + test to get the build going again. + + Relates #34683 + Relates #32966 + +commit f854330e069e5046b80538f6050eea58c07e33b1 +Author: Julie Tibshirani +Date: Mon Oct 22 11:54:04 2018 -0700 + + Make sure to use the type _doc in the REST documentation. (#34662) + + * Replace custom type names with _doc in REST examples. + * Avoid using two mapping types in the percolator docs. + * Rename doc -> _doc in the main repository README. + * Also replace some custom type names in the HLRC docs. + +commit d9817461420de4be034895e208c5292c7aaf7c19 +Author: Zachary Tong +Date: Mon Oct 22 13:15:45 2018 -0400 + + [Docs] clarification about cardinality accuracy (#34616) + + Adds a bit more clarification about how accuracy is dependent + on the dataset in question. + + Closes #18231 + +commit c344293aed997ab6f844f7ad9dab15ff1b1d9d55 +Author: Jay Modi +Date: Mon Oct 22 10:12:37 2018 -0600 + + Security: don't call prepare index for reads (#34568) + + The security native stores follow a pattern where + `SecurityIndexManager#prepareIndexIfNeededThenExecute` wraps most calls + made for the security index. The reasoning behind this was to check if + the security index had been upgraded to the latest version in a + consistent manner. However, this has the potential side effect that a + read will trigger the creation of the security index or an updating of + its mappings, which can lead to issues such as failures due to put + mapping requests timing out even though we might have been able to read + from the index and get the data necessary. + + This change introduces a new method, `checkIndexVersionThenExecute`, + that provides the consistent checking of the security index to make + sure it has been upgraded. That is the only check that this method + performs prior to running the passed in operation, which removes the + possible triggering of index creation and mapping updates for reads. + + Additionally, areas where we do reads now check the availability of the + security index and can short circuit requests. Availability in this + context means that the index exists and all primaries are active. + + This is the fixed version of #34246, which was reverted. + + Relates #33205 + +commit 5dd79bf58c0e0de9fb269c8d60d062a6352553cc +Author: Lee Hinman +Date: Mon Oct 22 09:55:00 2018 -0600 + + Make accounting circuit breaker settings dynamic (#34372) + + * Make accounting circuit breaker settings dynamic + + These missed the original property making them dynamic. This fixes the issue so + these can now be set at any time. + + Resolves #34368 + +commit fbb9ac34f97f4021d4e0e4dddbd19ae334dd5c4d +Author: Julie Tibshirani +Date: Mon Oct 22 08:46:11 2018 -0700 + + Deprecate type exists requests. (#34663) + +commit 52fc502b7e45e45e3fb3af0607696ffb943c77b9 +Author: Jason Tedor +Date: Mon Oct 22 11:23:33 2018 -0400 + + Fix the casing in the names of some CCR classes + + We should be consistent here. We were already using the casing "Ccr" and + this is the preferred casing for Java class names. This commit adjusts + the names of some classes that were using the casing "CCR" to be "Ccr". + +commit 6d6ac74a08fbe0535679bbd3d349bf20310ccf96 +Author: Yannick Welsch +Date: Mon Oct 22 17:20:12 2018 +0200 + + Zen2: Fail fast on disconnects (#34503) + + Integrates the failure detectors with the Connection lifecycle, to fail nodes as soon as: + - a leader detects one of his followers disconnecting. + - a follower detects its leader disconnecting. + +commit e869f9a78c6d6dcdae83d1925514f0772fff3be8 +Author: Christoph Büscher +Date: Mon Oct 22 17:15:21 2018 +0200 + + [Docs] Update synonym-tokenfilter.asciidoc (#34706) + + Remove ugly double-dot. + +commit 66ebcdc112d5e38f1d960e63e110fc83544324d6 +Author: Jason Tedor +Date: Mon Oct 22 11:14:39 2018 -0400 + + Enable using defined setups from test setup in docs (#34681) + + In the docs tests, we have pre-defined setups in the build.gradle file, + and we can also define test setup sections within the doc page + itself. Alas, these two are incompatible in that if you try to use a + pre-defined setup alongside a test setup section, the pre-defined setup + will be silently ignored. This commit enables pre-defined setup sections + to be used together with test setup sections. The ordering here is that + pre-defined setup sections will be executed first, followed by the test + setup section. + +commit 243335e2ba23966f549b640572a40a9a3bdb5b23 +Author: Jason Tedor +Date: Mon Oct 22 11:14:27 2018 -0400 + + Allow set section in setup section of REST tests (#34678) + + This commit enables using a set section in the setup section of REST + tests. + +commit 7af19b8f81242d16b58599324c191ca1d97bff8d +Author: Jason Tedor +Date: Mon Oct 22 11:14:02 2018 -0400 + + Migrate wait for pending tasks helper to server (#34675) + + In some of our X-Pack REST tests we have to wait for pending tasks to + complete. We are now needing this functionality in ESRestTestCase for + the docs tests where we run against X-Pack features. This commit moves + the helper method that we have in X-Pack to ESRestTestCase, and removes + duplicate logic from waiting for rollup tasks to complete. + +commit 92e34732f5ca9590bf117cef81a2092cb11bb203 +Author: Martijn van Groningen +Date: Mon Oct 22 09:15:22 2018 +0200 + + [CCR] Remove ccr related metadata between tests for single node tests too + +commit b6750cf6c22f4efad1b58b36c7db694d0bbc02fc +Author: Martijn van Groningen +Date: Mon Oct 22 08:47:31 2018 +0200 + + [CCR] Muted tests + + Relates to #34696 + +commit f51301a1a691241b80e79e53555847c207647772 +Author: Martijn van Groningen +Date: Mon Oct 22 08:44:41 2018 +0200 + + [CCR] Moved integration test + +commit b816837d395928586d97cce83b46979e46f904f3 +Author: Martijn van Groningen +Date: Mon Oct 22 08:15:43 2018 +0200 + + [CCR] Always remove persistent tasks metadata between tests and + better handle assertion errors between tests. + +commit e562afad6958a2bbe4da1605ce677f4e0d93110e +Author: Jason Tedor +Date: Sun Oct 21 19:13:53 2018 -0400 + + Awaits fix PainlessDomainSplitIT#testIsolated + + This test fails reliably with a compilation error. This commit awaits + fix the test. + +commit 0577703183ec3e341b88ff3ff43410683f5bbad9 +Author: Jason Tedor +Date: Sun Oct 21 13:16:15 2018 -0400 + + Revert "ingest: processor stats (#34202)" + + This reverts commit 65677296000558f0d848b67b19d5d588110af5bb. + +commit bf5f0af49151fbf9bc335a17cb08d730a2946831 +Author: Jason Tedor +Date: Sun Oct 21 06:23:42 2018 -0400 + + Use trial license in docs tests (#34673) + + This commit switches to using a trial license in the docs tests that run + on the default distribution. This is needed so that docs tests can be + executed against non-basic features. + +commit 222652dfce675cb47d6eca42d5668ca3b35a2374 +Author: Ryan Ernst +Date: Sat Oct 20 16:33:49 2018 -0700 + + Scripting: Convert script fields to use script context (#34164) + + This commit removes the use of SearchScript for script fields and adds + a new FieldScript. + +commit 7ab464807d8b31ce48a5fe37c43d922e325e0006 +Author: Nhat Nguyen +Date: Sat Oct 20 18:19:38 2018 -0400 + + TEST: Mute testDedupByPrimaryTerm + + Should be fixed by #34667 + +commit 65677296000558f0d848b67b19d5d588110af5bb +Author: Jake Landis +Date: Sat Oct 20 16:01:01 2018 -0500 + + ingest: processor stats (#34202) + + This change introduces stats per processors. Total, time, failed, + current are currently supported. All pipelines will now show all + top level processors that belong to it. Failure processors are not + displayed, however, the time taken to execute the failure chain is part + of the stats for the top level processor. + + The processor name is the type of the processor, ordered as defined in + the pipeline. If a tag for the processor is found, then the tag is + appended to the type. + + Pipeline processors will have the pipeline name appended to the name of + the name of the processors (before the tag if one exists). If more + then one pipeline is used to process the document, then each pipeline + will carry its own stats. The outer most pipeline will also include the + inner most pipeline stats. + + Conditional processors will only included in the stats if the condition evaluates + to true. + +commit dca452f0433864677481cc22e75823b760de58bc +Author: Jason Tedor +Date: Sat Oct 20 16:01:32 2018 -0400 + + Fix JDK references in contributing docs (#34669) + + After we bumped the JDK compiler minimum requirements to JDK 11, we + missed updating the contributing docs to point to JDK 11 instead of JDK + 10. This commit fixes that. + + Additionally, when we stopped requiring JDK 7 to run the reindex from + old tests, we did not update the contributing docs to reflect this. This + commit also fixes that. + +commit 19cc4439d78c30972c4cab76178a57dd82c6b508 +Author: Jason Tedor +Date: Sat Oct 20 15:57:53 2018 -0400 + + Fix cluster formation BWC versions + + We support JDK 10 through the 6.3 and 6.4 series. This commit adjusts + the BWC logic when starting nodes during tests to account for this. + +commit aebb855f38f39ccd49a872e2d7d40c2dd9749d80 +Author: Jason Tedor +Date: Sat Oct 20 12:11:41 2018 -0400 + + Add missing word to remote clusters docs + + This commit adds a missing word in the initial section of the remote + clusters docs. + +commit aca8c705ae3b63be4a02a0b04db93352a7d2854d +Author: Jason Tedor +Date: Sat Oct 20 11:18:27 2018 -0400 + + Fix name of remote clusters reference + + This commit fixes the name of the reference to the remote clusters + module on the modules page by adding a missing character. + +commit 8bed0c68e21babdccff5af8104035c23a42b42be +Author: Jason Tedor +Date: Sat Oct 20 11:16:39 2018 -0400 + + Fix remote clusters section link + + This commit fixes the remote clusters section link by changing some + underscores to dashes. + +commit 77d23a2007ac2bfe2fe9ac81b00812f42300961b +Author: Jason Tedor +Date: Sat Oct 20 10:28:25 2018 -0400 + + Fix remote clusters section link + + This fixes a link to the configuring remote clusters section of the + docs. + +commit 19ee5da0371f252b20c4d959df3ab043d81d8d8a +Author: Jason Tedor +Date: Sat Oct 20 09:35:24 2018 -0400 + + Fix reference to remote clusters configuration + + This commit fixes a reference to configuring a remote cluster in the + cross-cluster search and security docs. + +commit ab5a0b15f6205ed426895e2805d8e4fb56fdd657 +Author: Jason Tedor +Date: Sat Oct 20 09:04:48 2018 -0400 + + Fix formatting of remotes clusters module section + + This commit fixes the formatting of the remote clusters module section + on the modules page. We add a missing newline and wrap the columns at + 80. + +commit 91dbb2295099185064891c01356c72ea30fcf659 +Author: Jason Tedor +Date: Sat Oct 20 09:02:00 2018 -0400 + + Fix reference to remote clusters docs + + This commit fixes a reference to the remote clusters docs from the + modules page. + +commit b2dc58450e84817ae0664805aa391317d3e9027d +Author: Jason Tedor +Date: Sat Oct 20 08:33:59 2018 -0400 + + Separate remote clusters docs from CCS (#34612) + + With remote clusters taking on a larger role, we have make the + infrastructure more generic than being tied to cross-cluster search + (CCS). We want to refer to the remote clusters configuration in the + cross-cluster replication (CCR) docs. Yet, these docs are still tied to + CCS. This commit extracts the remote clusters docs from CCS (with some + wording changes to make them more general) so that we can refer to them + in the CCR docs. + +commit 9fb2f7cc20fab915ee57f79bba641efda8fb17f7 +Author: Yannick Welsch +Date: Sat Oct 20 12:11:36 2018 +0200 + + HLRC: Delete role API (#34620) + + Adds the "Delete role" API to the high-level REST client. + +commit adc3816e3288b1a9e5bf1d01a82773a722d89aa5 +Author: Alpar Torok +Date: Sat Oct 20 11:14:41 2018 +0300 + + Enable the Gradle daemon (#34545) + +commit d90b6730c7dbbeeb5317245b95d4455bba1453ce +Author: Nhat Nguyen +Date: Fri Oct 19 21:10:13 2018 -0400 + + CCR: Following primary should process NoOps once (#34408) + + This is a follow-up for #34288. + + Relates #34412 + +commit ba87c543c0ec034203c6236db50ece4ae6a0ee14 +Author: Jim Ferenczi +Date: Sat Oct 20 01:05:48 2018 +0200 + + [TEST] Fix sporadic failures in CompletionSuggestSearchIT#testTiebreak + + Relates #34508 + +commit 48ab6d7c92180285f5777ebd157463163c5600f9 +Author: Jason Tedor +Date: Fri Oct 19 15:11:48 2018 -0400 + + Switch old reindex tests to use use JDK 8 (#34646) + + This commit switches the old reindex tests to use JDK 8. This enables us + to not require that JDK 7 be installed, easing the burden on developers. + +commit 630d5514a54be1527b695c4f33123902ee3633c5 +Author: Nhat Nguyen +Date: Fri Oct 19 15:05:22 2018 -0400 + + CCR/TEST: Adjust testFailOverOnFollower + + CI passed but the result is outdated after PR #34366 was merged. + +commit 842bf554971a466c7277e415d73e548d2cabd7e9 +Author: Jack Conradson +Date: Fri Oct 19 11:52:15 2018 -0700 + + [Docs] Fixes a bug in the rest-client watcher docs related to + cross-link issue. + +commit bfd24fc030edeeba654bd6bda1142bbb861d8ea2 +Author: David Turner +Date: Fri Oct 19 19:24:54 2018 +0100 + + [Zen2] Reconfigure cluster as its membership changes (#34592) + + As master-eligible nodes join or leave the cluster we should give them votes or + take them away, in order to maintain the optimal level of fault-tolerance in + the system. #33924 introduced the `Reconfigurator` to calculate the optimal + configuration of the cluster, and in this change we add the plumbing needed to + actually perform the reconfigurations needed as the cluster grows or shrinks. + +commit bd92a28cfc664b9e4b1407978fe14744d445ba3d +Author: Nhat Nguyen +Date: Fri Oct 19 13:56:00 2018 -0400 + + CCR: Replicate existing ops with old term on follower (#34412) + + Since #34288, we might hit deadlock if the FollowTask has more fetchers + than writers. This can happen in the following scenario: + + Suppose the leader has two operations [seq#0, seq#1]; the FollowTask has + two fetchers and one writer. + + 1. The FollowTask issues two concurrent fetch requests: {from_seq_no: 0, + num_ops:1} and {from_seq_no: 1, num_ops:1} to read seq#0 and seq#1 + respectively. + + 2. The second request which fetches seq#1 completes before, and then it + triggers a write request containing only seq#1. + + 3. The primary of a follower fails after it has replicated seq#1 to + replicas. + + 4. Since the old primary did not respond, the FollowTask issues another + write request containing seq#1 (resend the previous write request). + + 5. The new primary has seq#1 already; thus it won't replicate seq#1 to + replicas but will wait for the global checkpoint to advance at least + seq#1. + + The problem is that the FollowTask has only one writer and that writer + is waiting for seq#0 which won't be delivered until the writer completed. + + This PR proposes to replicate existing operations with the old primary + term (instead of the current term) on the follower. In particular, when + the following primary detects that it has processed an process already, + it will look up the term of an existing operation with the same seq_no + in the Lucene index, then rewrite that operation with the old term + before replicating it to the following replicas. This approach is + wait-free but requires soft-deletes on the follower. + + Relates #34288 + +commit 94bde37bcfea3cdf5c69857a7a7f3ef6d4e8fa0b +Author: Igor Motov +Date: Fri Oct 19 13:53:54 2018 -0400 + + Geo: Don't flip longitude of envelopes crossing dateline (#34535) + + When a envelope that crosses the dateline is specified as a part of + geo_shape query is parsed it shouldn't have its left and right points + flipped. + + Fixes #34418 + +commit fba5d39bbba8fafa17b1f06b61f124fc91cba159 +Author: Jim Ferenczi +Date: Fri Oct 19 19:46:55 2018 +0200 + + Fix completion suggester's score tie-break (#34508) + + The shard suggestion sort uses a different tie-break than the one that is used + to merge different shards responses. The former uses the internal document identifier + when scores are the same whereas the latter compares the surface form first. + Because of this discrepancy some suggestion outputs are linked to the wrong documents + because the merge sort reorders the shard suggestions differently. This change + fixes this bug by duplicating the Lucene collector in order to be able to apply the + same tiebreak strategy than the merge sort. This logic will be removed when + https://issues.apache.org/jira/browse/LUCENE-8529 is fixed. + + Closes #34378 + +commit b0d4ba2431fe0bccb511af6fdb21a937282c9ffc +Author: Benjamin Trent +Date: Fri Oct 19 12:25:42 2018 -0500 + + HLRC: ML %s/Boolean is/Boolean get/g (#34641) + +commit 423d085bed9d3e140ca4a0a822b9ef4941081e82 +Author: Vladimir Dolzhenko +Date: Fri Oct 19 19:05:48 2018 +0200 + + HLRC XPack Protocol clean up: Licence, Misc (#34469) + + + Relates #34451 + +commit 67652b53559207c9cf49c9d842887349e09d92f5 +Author: Julie Tibshirani +Date: Fri Oct 19 09:47:34 2018 -0700 + + Remove references to multiple types in the search documentation. (#34625) + +commit 1100942455a131cadcde8510397044f2cba43f50 +Author: Jack Conradson +Date: Fri Oct 19 09:42:57 2018 -0700 + + [HLRC] Add Start/Stop Watch Service APIs. (#34317) + +commit 90ca5b1fdeede0aa83d51892ff7ff63ff715333a +Author: Nhat Nguyen +Date: Fri Oct 19 12:38:06 2018 -0400 + + Fill LocalCheckpointTracker with Lucene commit (#34474) + + Today we rely on the LocalCheckpointTracker to ensure no duplicate when + enabling optimization using max_seq_no_of_updates. The problem is that + the LocalCheckpointTracker is not fully reloaded when opening an engine + with an out-of-order index commit. Suppose the starting commit has seq#0 + and seq#2, then the current LocalCheckpointTracker would return "false" + when asking if seq#2 was processed before although seq#2 in the commit. + + This change scans the existing sequence numbers in the starting commit, + then marks these as completed in the LocalCheckpointTracker to ensure + the consistent state between LocalCheckpointTracker and Lucene commit. + +commit c476f918110fe04e1da9cd46c5b3409fba5b96ba +Author: David Kyle +Date: Fri Oct 19 16:29:16 2018 +0100 + + [TEST][ML] Reinstate date parsing test (#34634) + + The test was muted due to a bug in an early access build of Java 11 + https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8206980 + +commit 52104aac27c8e71e6e738ff624537e63b7117317 +Author: Costin Leau +Date: Fri Oct 19 16:44:33 2018 +0300 + + SQL: Introduce support for NULL values (#34573) + + Make SQL aware of missing and/or unmapped fields treating them as NULL + Make _all_ functions and operators null-safe aware, including when used + in filtering or sorting contexts + Add missing and null-safe doc value extractor + Modify dataset to have null fields spread around (in groups of 10) + Enforce missing last and unmapped_type inside sorting + Consolidate Predicate templating and declaration + Add support for Like/RLike in scripting + Generalize NULLS LAST/FIRST + Introduce early schema declaration for CSV spec tests: to keep the doc + snippets in place (introduce schema:: prefix for declaration) + upfront. + + Fix #32079 + +commit 3de266e3cf43d5a41b10f277156cf849fbaddc83 +Merge: e13ce66a3c2 fe623acf66c +Author: David Turner +Date: Fri Oct 19 14:30:07 2018 +0100 + + Merge branch 'master' into zen2 + +commit 910e97422faab290d5cde39e486ecb61717779f1 +Author: Armin Braun +Date: Fri Oct 19 14:04:49 2018 +0100 + + HLRC: Add ILM Retry (#33990) + + * HLRC: Add ILM Retry + + * Relates #33100 + +commit 84ef91529cf7dd9ec8c78f299d8c89dfd64a2106 +Merge: 9cb0bb8b9f4 3036ab10483 +Author: Colin Goodheart-Smithe +Date: Fri Oct 19 13:24:04 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 4236358f5d53ab4f0d8457faacb19983b6b1ab82 +Author: Benjamin Trent +Date: Fri Oct 19 07:23:25 2018 -0500 + + Rollup add default metrics to histo groups (#34534) + + * Rollup: Adding default metrics for histo group timefield (#34379) + + * Rollup: Adding default histo metrics and tests + + * fixing failing client side test with new default values + + * Adding HLRC docs for default values + + * Addressing PR comments + + * Removing value_count default agg + + * Updating docs for rollups + + * Minor interval change + +commit 3036ab10483d4fa383fd8b149e38827ec437c629 +Author: Christophe Bismuth +Date: Fri Oct 19 13:57:41 2018 +0200 + + Don't omit default values when updating routing exclusions (#33638) + + Exclusion setting `cluster.routing.allocation.exclude._host` default value + is an empty string. + + When an exclusion setting is sent with a null value the + o.e.c.s.Setting#innerGetRaw API return an empty string (probably to + avoid a NullPointerException to be raised). + + The o.e.c.r.a.d.FilterAllocationDecider class is developed to omit + updates of default values for exclusion setting. + + That's why a null exclusion setting value is translated to an empty + string which is equals to the exclusion default value which is + configured to be ignored. + + A simple fix would be to not omit default values for exclusion setting + and keep the NullPointerException guard. This is the purpose of this + commit. + + Closes #32721 + +commit aeb3cda585e56a93dec2c88037942f00f94687bb +Author: Albert Zaharovits +Date: Fri Oct 19 14:29:37 2018 +0300 + + [TEST] Fix GetUserPrivilegesResponse#Indices(StreamInput) constr + + Closes #34631 + +commit 7b49beb9b0e6f4a5a62479b67d6514f738356150 +Author: Jim Ferenczi +Date: Fri Oct 19 13:33:19 2018 +0200 + + Fix threshold frequency computation in Suggesters (#34312) + + The `term` and `phrase` suggesters have different options to filter candidates + based on their frequencies. The `popular` mode for instance filters candidate + terms that occur in less docs than the original term. However when we compute this threshold + we use the total term frequency of a term instead of the document frequency. This is not inline + with the actual filtering which is always based on the document frequency. This change fixes + this discrepancy and clarifies the meaning of the different frequencies in use in the suggesters. + It also ensures that the threshold doesn't overflow the maximum allowed value (Integer.MAX_VALUE). + + Closes #34282 + +commit c1c447a4cfd827923cf35bd63fdfeee8710778a2 +Author: Christoph Büscher +Date: Fri Oct 19 12:59:23 2018 +0200 + + Check stemmer language setting early (#34601) + + Currently the StemmerTokenFilterFactory checks the validity of the language + setting only when the first TokenStream is processed. Instead we should throw an + error earlier at mapping creation time. This change adds a check to the + StemmerTokenFilterFactory constructor that checks for a valid `language` setting + by trying to create a new TokenStream from an empty input stream. This will + throw errors about wrong language settings early on. + + Closes #34170 + +commit 59033e0e4526c8aa74c119b6a7a87e118953e7e8 +Author: Ioannis Kakavas +Date: Fri Oct 19 13:58:24 2018 +0300 + + [TEST] Add SAML Redirect Signing test (#34562) + + Add a test to verify that we generate correct signatures for our + SAML2 Single Logout requests when using the redirect binding. + +commit fe623acf66cbf4b86e0ce0be153cae87c389877a +Author: markharwood +Date: Fri Oct 19 09:33:59 2018 +0100 + + Docs - removed experimental/beta markers from adjacency matrix aggregation (#34599) + +commit 2a413abb0b87cec06fc02b5586c13acc89e5a8c5 +Author: markharwood +Date: Fri Oct 19 09:32:02 2018 +0100 + + Docs - remove experimental marker from significant_text aggregation (#34598) + +commit dbb6fe58faf27598d7bb735365710903c1ef137d +Author: Daniel Mitterdorfer +Date: Fri Oct 19 10:13:13 2018 +0200 + + Remove hand-coded XContent duplicate checks + + With this commit we cleanup hand-coded duplicate checks in XContent + parsing. They were necessary previously but since we reconfigured the + underlying parser in #22073 and #22225, these checks are obsolete and + were also ineffective unless an undocumented system property has been + set. As we also remove this escape hatch, we can remove the additional + checks as well. + + Closes #22253 + Relates #34588 + +commit e498b7d4373eb5696d45ed35df6fd1cfc3f1b38e +Author: Alexander Reelsen +Date: Fri Oct 19 10:02:45 2018 +0200 + + Core: Parse floats in epoch millis parser (#34504) + + In order to stay BWC compatible with joda time, the epoch millis date + formatter needs to parse dates with a dot like `123.45`. This + adds this functionality for the epoch millis parser in the same way as + for the epoch seconds parser. It also adds support for scientific + notations like `1.0e3` and fixes parsing of negative values for epoch + seconds and epoch millis. + +commit 4f7895800e17b67aafec7127d7dfc61acd8598b4 +Author: Christoph Büscher +Date: Fri Oct 19 09:50:45 2018 +0200 + + Remove unused methods in ValueType (#34624) + + The removed methods seem unused in the rest of the project. + +commit 670ccfb85383f649168c613125466fefc9544019 +Author: Tim Vernum +Date: Fri Oct 19 17:54:01 2018 +1100 + + Handle missing user in user privilege APIs (#34575) + + For user/_has_privileges and user/_privileges, handle the case where + there is no user in the security context. This is likely to indicate + that the server is running with a basic license, in which case the + action will be rejected with a non-compliance exception (provided + we don't throw a NPE). + + The implementation here is based on the _authenticate API. + + Resolves: #34567 + +commit 56d4f6971845fcf30f08be70f81ade1e4fe3d6b8 +Author: Martijn van Groningen +Date: Fri Oct 19 07:59:20 2018 +0200 + + Renamed remaining leader_cluster_alias / cluster_alias to leader_cluster + +commit 44b461aff2bf5de6517cb8ef3c43ca61e2286cef +Author: Martijn van Groningen +Date: Fri Oct 19 07:41:46 2018 +0200 + + [CCR] Make leader cluster a required argument. (#34580) + + This change makes it no longer possible to follow / auto follow without + specifying a leader cluster. If a local index needs to be followed + then `cluster.remote.*.seeds` should point to nodes in the local cluster. + + Closes #34258 + +commit 39a61633162248341931e1679113d1fc8f4927fb +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Oct 19 13:46:26 2018 +1100 + + [HLRC] Add support for Delete role mapping API (#34531) + + Building on expression dsl and create role mapping API, this + commit adds support for delete role mapping API to high level client. + +commit 2283a337b979f9d80457b8985d56c2923ea95380 +Author: Tim Vernum +Date: Fri Oct 19 11:40:53 2018 +1100 + + [TEST] Fix bit indexing in test mutator + + There are 2 hard problems in computer science: cache invalidation, + naming things, and off-by-1 errors. + -- Leon Bambrick (@secretGeek) + + Fixes: #34610 + +commit 7cc3ac0fc475f8beecd4a426a20ec11fcf8c49c1 +Author: Christoph Büscher +Date: Fri Oct 19 00:44:37 2018 +0200 + + [Docs] Add _cache/clear parameter examples (#34614) + + Add example for selectively clearing just the request, query or fielddata cache + and for selectively clearing the cache for specific fields. + + Closes #34287 + +commit 45546e71c227bba378d8ce5fe7a6a8875ca3ed39 +Author: Zachary Tong +Date: Thu Oct 18 17:12:38 2018 -0400 + + Add GetRollupCaps API to high level rest client (#32880) + + Adds GetRollupCaps API to the HLRC, and tweaks some of the + Caps objects to be immutable. Also various style tweaks + +commit fa973007b80da8ba17856be882f425069de3b8e7 +Author: Benjamin Trent +Date: Thu Oct 18 16:11:36 2018 -0500 + + [DOCS] Adds new lazy ml node setting (#34600) + + * Adding new xpack.ml.max_lazy_ml_nodes setting to docs + + * Fixing docs, making it clearer what the setting does + + * Adding note about external process need + +commit 9cb0bb8b9f41ef9fa5c46e8fda608fcfd162dd2e +Author: Gordon Brown +Date: Thu Oct 18 13:33:33 2018 -0600 + + Rework ILM build to separate integration tests (#34617) + + Having integration tests separated from the unit tests in the qa + directory works much more smoothly with our testing infrastructure, + matches what other plugins do, and tests in a more "real" deployment + scenario by having all plugins installed. + +commit 2c0f67318a90af20135fc96ad48e043e6e585710 +Author: Nik Everett +Date: Thu Oct 18 15:24:02 2018 -0400 + + Docs: Reenable rollup docs tests (#34564) + + We'd disabled them because we didn't have a way to clean up after each + test. I implemented #34342 which adds the clean ups so now we can + re-enable the tests. + + In the `setup` sections we have to use `raw` requests instead of + `x-pack` requests because we don't have the json config for x-pack. + + Closes #33319 + +commit ca51fb687393ede4b3ed27145efbd46a3ae883b0 +Author: Zachary Tong +Date: Thu Oct 18 12:12:17 2018 -0400 + + [Rollup] Add support for date histo `format` (#34537) + + Adds support for query-time formatting of the date histo keys + when executing a rollup search. + + Closes #34391 + +commit fe41b2cc0be1f2a503f0ca9844b9119c68c441fb +Author: Ioannis Kakavas +Date: Thu Oct 18 18:22:46 2018 +0300 + + Change wording for FIPS 140-2 (#34471) + + Changes wording in the FIPS 140-2 related documentation. + + Co-authored-by: derickson + +commit 6ccaa83d2aa78431ac7e0ab5f2946224664a6d27 +Author: Benjamin Trent +Date: Thu Oct 18 09:57:20 2018 -0500 + + Fixing line lengths in murmur3 and hdfs plugins (#34603) + +commit f11a23be2f8cbb3241f63f66df4aabed83ca7080 +Author: Igor Motov +Date: Thu Oct 18 10:49:23 2018 -0400 + + Fix line longer than 140 characters in Murmur3FieldMapper + +commit 6c07d105f3c0cb27610affa463910e96e3e1609f +Author: Nik Everett +Date: Thu Oct 18 09:09:09 2018 -0400 + + Amazon: Wrap at 140 columns (#34495) + + Applies our standard column wrapping to the `discovery-ec2` and + `repository-s3` plugins. + +commit e13ce66a3c2c6620576c6a1852699953a678381e +Author: David Turner +Date: Thu Oct 18 13:19:27 2018 +0100 + + [Zen2] Calculate optimal cluster configuration (#33924) + + We wish to commit a cluster state update after having received a response from + more than half of the master-eligible nodes in the cluster. This is optimal: + requiring either more or fewer votes than half harms resilience. For instance + if we have three master nodes then, we want to be able to commit a cluster + state after receiving responses from any two nodes; requiring responses from + all three is clearly not resilient to the failure of any node, and if we could + commit an update after a response from just one node then that node would be + required for every commit, which is also not resilient. + + However, this means we must adjust the configuration (the set of voting nodes + in the cluster) whenever a master-eligible node joins or leaves. The + calculation of the best configuration for the cluster is the job of the + Reconfigurator, introduced here. + +commit 0f8d05f2d0fd40510faf5428235d43873eb8ad1d +Author: David Roberts +Date: Thu Oct 18 12:43:41 2018 +0100 + + [TEST] Reduce forecast disk space requirement for tests (#34552) + + The setting that reduces the disk space requirement + for the forecasting integration tests was accidentally + removed in #31757 when files were moved around. This + change simply adds back the setting that existed before + that. + +commit 60b8118d3a9036d2b828f967863049be08d018f8 +Author: Andrei Stefan +Date: Thu Oct 18 13:33:01 2018 +0300 + + A constant can be used outside aggregation only queries (#34576) + + A constant can now be used outside aggregation only queries. + Don't skip an ES query in case of constants-only selects. + Loosen the binary pipe restriction of being used only in aggregation queries. + Fixes https://github.com/elastic/elasticsearch/issues/31863 + +commit 0d62f6102cbad45e4d7cdb84744b8cde7bef0b49 +Author: Martijn van Groningen +Date: Thu Oct 18 12:11:48 2018 +0200 + + [CCR] Split cluster alias from leader index field into its own field in follow APIs (#34366) + +commit 4e36251835a14b0b8f02906bde3721298175d3db +Author: Alexander Reelsen +Date: Thu Oct 18 11:54:50 2018 +0200 + + Watcher: Validate email adresses when storing a watch (#34042) + + Right now, watches fail on runtime, when invalid email addresses are + used. + + All those fields can be checked on parsing, if no mustache is used in + any email address template. In that case we can return immediate + feedback, that invalid email addresses should not be specified when + trying to store a watch. + +commit 7bcf49631564c22136abd72b8646e0e6c360b471 +Author: Christoph Büscher +Date: Thu Oct 18 11:23:53 2018 +0200 + + [Tests] Correct map lookup in ReplicationTrackerTests (#34565) + +commit df3726dc9bdcf6a30618a6d775bc8200e1a4a221 +Author: Albert Zaharovits +Date: Thu Oct 18 12:06:47 2018 +0300 + + Rename and deprecated `prefix` logfile audit setting names (#34475) + + The logfile audit log format is no longer formed by prefix fields followed + by key value fields, it is all formed by key value fields only (JSON format). + + Consequently, the following settings, which toggled some of the prefix + fields, have been renamed: + audit.logfile .prefix.emit_node_host_address + audit.logfile .prefix.emit_node_host_name + audit.logfile .prefix.emit_node_name + +commit 6874ee8938a280dcb91eeef3077a75e5a8a8e51f +Author: Ioannis Kakavas +Date: Thu Oct 18 10:15:05 2018 +0300 + + Remove deprecated setting from test + + Replace xpack.security.transport.ssl.truststore.password deprecated + setting with xpack.security.transport.ssl.truststore.secure_password + in the HLRC integTestCluster. + +commit 3621277969fec25ad1a04355dd6f24597d8aa492 +Author: Vladimir Dolzhenko +Date: Wed Oct 17 17:43:26 2018 +0200 + + eliminate NPE on closing restHighLevelClient (if test suite skips all test cases) + + (cherry picked from 277704c5972148ee62900e781ac64b061aa4b767) + +commit e41c5cb47aae98f85347ee5b37f77dbf30072813 +Author: Tim Vernum +Date: Thu Oct 18 18:03:54 2018 +1100 + + Fix compilation issue due to 47e9082 and 9200e15 + + 47e9082 removed a method that 9200e15 made use of. + +commit 47e9082bb414158d6d42336fd339e24a3dbf362b +Author: Tim Vernum +Date: Thu Oct 18 17:28:43 2018 +1100 + + Replace Streamable w/ Writeable in RoleDescriptor (#34544) + + This commit replaces usage of Streamable with Writeable within the + RoleDescriptor class (and inner classes). + + Relates: #34389 + +commit 9200e15b74d92593aea0c157741739c12247f6c4 +Author: Tim Vernum +Date: Thu Oct 18 14:09:04 2018 +1100 + + Add get-user-privileges API (#33928) + + This API is intended as a companion to the _has_privileges API. + It returns the list of privileges that are held by the current user. + + This information is difficult to reason about, and consumers should + avoid making direct security decisions based solely on this data. + For example, each of the following index privileges (as well as many + more) would grant a user access to index a new document into the + "metrics-2018-08-30" index, but clients should not try and deduce + that information from this API. + - "all" on "*" + - "all" on "metrics-*" + - "write" on "metrics-2018-*" + - "write" on "metrics-2018-08-30" + + Rather, if a client wished to know if a user had "index" access to + _any_ index, it would be possible to use this API to determine whether + the user has any index privileges, and on which index patterns, and + then feed those index patterns into _has_privileges in order to + determine whether the "index" privilege had been granted. + + The result JSON is modelled on the Role API, with a few small changes + to reflect how privileges are modelled when multiple roles are merged + together (multiple DLS queries, multiple FLS grants, multiple global + conditions, etc). + +commit d445785f1ac5e84e635eccf625b3a483e6ff081b +Author: Ryan Ernst +Date: Wed Oct 17 15:54:21 2018 -0700 + + Scripting: Convert domainSplit function for ML to whitelist (#34426) + + This commit moves the definition of domainSplit into java and exposes it + as a painless whitelist extension. The method also no longer needs + params, and version which ignores params is added and deprecated. + +commit 09067c89428733e6a093794025cc7e6e200b598e +Merge: 90f7cec7a5b 87345403453 +Author: Tal Levy +Date: Wed Oct 17 15:37:11 2018 -0700 + + Merge remote-tracking branch 'upstream/master' into index-lifecycle + +commit 873454034536bfd3de9447de2c736d329018d8a7 +Author: Ryan Ernst +Date: Wed Oct 17 15:16:13 2018 -0700 + + Ensure map keys cannot be self referencing (#34569) + + This commit improves self reference checking to map keys, as well as + adds it to ingest script processing. + +commit 9be87adb9524d6547f354e9a342aa1abf945ef89 +Author: Jason Tedor +Date: Wed Oct 17 18:00:17 2018 -0400 + + Increment settings version when upgrading index (#34566) + + When we upgrade an index, we set the settings version upgraded + setting. This should be considered a settings change, and therefore we + need to increment the settings version. This commit addresses that. + +commit eb36f10394a8c0a6305d3b89bd0cf26dfe09bfda +Author: Nhat Nguyen +Date: Wed Oct 17 17:37:52 2018 -0400 + + TEST: Capture replication targets when replication group ready (#34407) + + Today, WriteReplicationAction uses a set of replication targets directly + from the primary shard of ReplicationGroup. It should be fine except + when we add/remove or promote a shard while a write action is executing. + We have encountered these two issues: + + 1. Replicas are not found in the replication targets. This happens + because we remove replicas but the WriteReplicationAction still uses the + old replication targets which include the removed replicas. + + 2. Access ReplicationGroup from a primary shard which hasn't activated + the primary-mode yet. This is because we won't activate the primary-mode + for a promoting shard after bumping the primary term which is executed + asynchronously. + + This commit captures the replication targets when the replication group + is ready and continue using those targets until we re-compute the new + targets after the group is changed. + + Closes #33457 + +commit a45626deb57536e9201a514f850a344b9243710b +Author: Nik Everett +Date: Wed Oct 17 16:17:25 2018 -0400 + + Analysis: Wrap at 140 columns (#34494) + + Applies our standard column width to all analysis plugins. + +commit 1452d55155d279269f9cfca8e7eccb4736799d95 +Author: Nik Everett +Date: Wed Oct 17 16:12:21 2018 -0400 + + Expressions: Wrap at 140 columns (#34493) + + Apply our standard line length to all of the java files in the + `lang-expression` plugin. + +commit 5d74f8085bc98e4427ffae83d82e563b65943206 +Author: Nik Everett +Date: Wed Oct 17 16:04:41 2018 -0400 + + SMB: Wrap at 140 columns (#34492) + + Applies our max column width to the store-smb project. + +commit b6aa42777adee26174ab86061fd2cd82c6c19f4a +Author: Nik Everett +Date: Wed Oct 17 15:54:35 2018 -0400 + + Search: Wrap lucene classes at 140 columns (#34491) + + Applies our line length guidance for all classes in the server in `lucene` + directories *except* `XMoreLikeThis`. The only long line in + `XMoreLikeThis` says "remove this when we upgrade to Lucene 5. Given + that we're on Lucene 8, this is a little terrifying and deserves another + look. + +commit 08d4bf6e8403419e9c561ebf7db9679bf8de40a0 +Author: Armin Braun +Date: Wed Oct 17 20:08:39 2018 +0100 + + TESTS: Remove Dead Code in Test Infra. (#34548) + + * None of this infrastructure is used + * Some redundant throws and resulting catch code removed + +commit 90f7cec7a5ba6545ed9134de8203bc4afce7727c +Merge: 7c7605d3d27 b0e98cbce2d +Author: Colin Goodheart-Smithe +Date: Wed Oct 17 18:22:23 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit b0e98cbce2d02695531254821d3d160e615373c0 +Author: Simon Willnauer +Date: Wed Oct 17 19:11:50 2018 +0200 + + Pass the host name on as `server_name` if proxy mode is on (#34559) + + In remote cluster setup if we see a configured proxy we should set + the seed nodes host name as the `server_name` to trigger SNI based + routing even for seed nodes. Since remote cluster connections are + plain TCP connections we have to set the host manually since the other + side can't take it from the request URL like in the HTTP case. + This also adds some more informative logging to remote cluster connection. + +commit 46c7b5ee6e53e33aa6f91c1cbd873810d319a400 +Author: jaymode +Date: Wed Oct 17 10:37:40 2018 -0600 + + Revert "Security: don't call prepare index for reads (#34246)" + + This reverts commit 0b4e8db1d3341d096e02bbdbb44fd8e5d221722a as some + issues have been identified with the changed handling of a primary + shard of the security index not being available. + +commit 51f38ddc0ca2bc8b71af8b41e0e212dbff14bdf8 +Author: Andrey Ershov +Date: Wed Oct 17 18:17:17 2018 +0200 + + Switch MetaDataStateFormat to Lucene directory abstraction (#33989) + + Switch MetaDataStateFormat to Lucene directory abstraction + + This commit switches MetaDataStateFormat class to Lucene directory abstraction to make it easier to test MetaDataStateFormat for different IO failures. + This commits also adds different IO failures tests to MetaDataStateFormatTests. + +commit 18aa1c1381b76769ed5ebaaf4b6840aa26301d75 +Author: Andy Bristol +Date: Wed Oct 17 08:02:04 2018 -0700 + + add start trial API to HLRC (#33406) + + Introduces client-specific request and response classes that do not + depend on the server + + The `type` parameter is named `licenseType` in the response class to be + more descriptive. The parts that make up the acknowledged-required + response are given slightly different names than their server-response + types to be consistent with the naming in the put license API + + Tests do not cover all cases because the integ test cluster starts up + with a trial license - this will be addressed in a future commit + +commit 139bbc3f0316f92974abccb4b38e2e39899a0006 +Author: Nik Everett +Date: Wed Oct 17 09:32:16 2018 -0400 + + Rollup: Consolidate rollup cleanup for http tests (#34342) + + This moves the rollup cleanup code for http tests from the high level rest + client into the test framework and then entirely removes the rollup cleanup + code for http tests that lived in x-pack. This is nice because it + consolidates the cleanup into one spot, automatically invokes the cleanup + without the test having to know that it is "about rollup", and should allow + us to run the rollup docs tests. + + Part of #34530 + +commit fb579d2d9ae8e6a598b2bf450c42b48caa6efae3 +Author: Benjamin Trent +Date: Wed Oct 17 08:30:15 2018 -0500 + + ML: Adding support for lazy nodes (#29991) (#34538) + +commit 2cccd4c669904e75095e2ed3f2eac1e65d19f0f0 +Author: Nik Everett +Date: Wed Oct 17 09:17:02 2018 -0400 + + LLREST: Wrap at 140 columns (#34489) + + Wraps a single line in a test to fit on 140 characters, removing the + only checkstyle suppression for the clients directory. + +commit 93bb24e1f83ecfdb3622c502c66e5387cec89134 +Merge: 303575f7422 030e8c8fe35 +Author: Andrey Ershov +Date: Wed Oct 17 14:37:53 2018 +0200 + + Merge branch 'master' into zen2 + +commit 030e8c8fe35b380d9a7024b5e4e58485b0882411 +Author: Nik Everett +Date: Wed Oct 17 08:13:36 2018 -0400 + + Docs: Tweak upgrade docs + + Tweak the upgrade instructions for moving from pre-6.3-with-x-pack to + post-6.3-default distribution. Specifically, you have to remove the + x-pack plugin before upgrading because 6.4 doesn't understand how to + remove it. + + Relates to #34307 + +commit e0a1803638baa54c114caed4ea623fbdb93ae0fb +Author: Tim Vernum +Date: Wed Oct 17 20:55:18 2018 +1100 + + Add Debug/Trace logging to token service (#34022) + + The token service has fairly strict validation and there are a range + of reasons why request may be rejected. + The detail is typically returned in the client exception / json body + but the ES admin can only debug that if they have access to detailed + logs from the client. + + This commit adds debug & trace logging to the token service so that it + is possible to perform this debugging from the server side if + necessary. + +commit e0cab14c6ecf5c00f28bb352732c02c762aa5ce3 +Author: Shaunak Kashyap +Date: Wed Oct 17 10:15:12 2018 +0100 + + Cleanup: removing unused class (#34541) + + * Cleanup: removing unused field in other similar classes + + * Removing unused class + +commit 3954d041a07dd2da97ce435c6fcd6c840694d185 +Author: Armin Braun +Date: Wed Oct 17 10:02:44 2018 +0100 + + SCRIPTING: Move sort Context to its Own Class (#33717) + + * SCRIPTING: Move sort Context to its own Class + +commit 7c7605d3d27f8f795f79a77383d397784f6ffe07 +Merge: fdb850735ab 2e5e4e1a690 +Author: Colin Goodheart-Smithe +Date: Wed Oct 17 09:29:06 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit fdb850735ab55d5decd281bcab94de88f8803a97 +Author: Tal Levy +Date: Tue Oct 16 23:39:48 2018 -0700 + + fix setting version on deleting unmaanged indices with wildcard + +commit 2e5e4e1a69032c756e5be27f32d302cbb556a804 +Author: Alpar Torok +Date: Wed Oct 17 07:57:30 2018 +0300 + + Switch to parametric CI jobs (#34179) + + Switch to parametric CI jobs. Given the changes in CI this also switches to running Gradle with the build java version only ( we used to also run it with run-time java version ) + +commit 335d268a66274b738ce76e311249a721fb10a1b4 +Author: Tal Levy +Date: Tue Oct 16 16:38:56 2018 -0700 + + fix timeout assertion in requestconvertertests + +commit 3a555da34d4e921f221482a07213d55d51a10e90 +Author: Tal Levy +Date: Tue Oct 16 15:43:10 2018 -0700 + + update version on ILM setting updates + +commit b3be96aeb9530322a4819f4112a18ccda33fa422 +Author: Armin Braun +Date: Tue Oct 16 22:48:28 2018 +0100 + + MINOR: Remove Deadcode in X-Pack Tests (#34511) + +commit fbe8dc014c73b833fb5eb5f95215b4286af05644 +Merge: 2846effc747 0b4e8db1d33 +Author: Tal Levy +Date: Tue Oct 16 13:58:53 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 2846effc747c3f5618c9ceff5a0b01658d0b2077 +Author: Tal Levy +Date: Tue Oct 16 12:18:54 2018 -0700 + + initial api reference docs for ILM (#33866) + + This adds the bulk of the ILM APIs in docs with examples + +commit 0b4e8db1d3341d096e02bbdbb44fd8e5d221722a +Author: Jay Modi +Date: Tue Oct 16 12:48:58 2018 -0600 + + Security: don't call prepare index for reads (#34246) + + The security native stores follow a pattern where + `SecurityIndexManager#prepareIndexIfNeededThenExecute` wraps most calls + made for the security index. The reasoning behind this was to check if + the security index had been upgraded to the latest version in a + consistent manner. However, this has the potential side effect that a + read will trigger the creation of the security index or an updating of + its mappings, which can lead to issues such as failures due to put + mapping requests timing out even though we might have been able to read + from the index and get the data necessary. + + This change introduces a new method, `checkIndexVersionThenExecute`, + that provides the consistent checking of the security index to make + sure it has been upgraded. That is the only check that this method + performs prior to running the passed in operation, which removes the + possible triggering of index creation and mapping updates for reads. + + Additionally, areas where we do reads now check the availability of the + security index and can short circuit requests. Availability in this + context means that the index exists and all primaries are active. + + Relates #33205 + +commit a93aefb4a43b03dfc3d31d140725f78d6484cc69 +Author: Simon Willnauer +Date: Tue Oct 16 20:22:32 2018 +0200 + + Assume that rollover datemath tests run on the same day. (#34527) + + in #28741 RolloverIT fails because we are cutting over to the + next day while the test executes. We assume that this doesn't happen + based on the assertions in the test. This adds a assumeTrue to ensure + we are at least 5 min away form a date-flip. + + Closes #28741 + +commit 2cc3caf5e2229f86628ed2a29c5565c68e8a2e90 +Author: Nik Everett +Date: Tue Oct 16 11:14:11 2018 -0400 + + Build: Drop missing checkstyle suppressions (#34490) + + This drops checkstyle suppressions that refer to files that don't exist + since those suppressions don't do anything other than make us feel bad. + + It also updates some suppressions to more closely match the path to the + file that they suppress. These suppressions are still needed but didn't + pass the "the file exists" test because they weren't precise. It is just + easier on future-me if they are precise. + +commit 80474e138f2a8a99b6226e29d38c1a155fcd6ae8 +Author: Jack Conradson +Date: Tue Oct 16 08:12:06 2018 -0700 + + HLRC: Add remove index lifecycle policy (#34204) + + This change adds the command RemoveIndexLifecyclePolicy to the HLRC. This uses the + new TimeRequest as a base class for RemoveIndexLifecyclePolicyRequest on the client side. + +commit 540dfcf23e58709e2e9f72da2daed479719f2988 +Author: Nik Everett +Date: Tue Oct 16 11:02:10 2018 -0400 + + JDBC: Fix artifactId in pom (#34478) + + We're publishing jdbc into our maven repo as though its artifactId is + `x-pack-sql-jdbc` but the pom listed the artifactId as `jdbc`. This + fixes the pom to line up with where we're publishing the artifact. + + Closes #34399 + +commit 9b5eaafc24930d09808b590da8e0598efb804472 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Tue Oct 16 16:46:14 2018 +0200 + + HLRC: Documentation examples cleanup (#34009) + + * Replace deprecated field `code` with `source` for stored scripts (#25127) + * Replace examples using the deprecated endpoint `{index}/{type}/_search` + with `{index}/_search` (#29468) + * Use a system property to avoid deprecation warnings after the Update + Scripts have been moved to their own context (#32096) + +commit 3e067123a132ef1e0e4e4efeaa16b73664624c26 +Author: Jason Tedor +Date: Tue Oct 16 10:45:05 2018 -0400 + + Remove dead methods from ChainIT + + This commit removes some unused methods from ChainIT. + +commit 303575f74223cc9075432b6a877370abe1568086 +Author: David Turner +Date: Tue Oct 16 15:29:47 2018 +0100 + + Fix up merge of master + +commit ea576a8ca2e78e5f8395d2ad07c45ed9627dd30e +Author: Armin Braun +Date: Tue Oct 16 15:28:40 2018 +0100 + + Disc: Move AbstractDisruptionTC to filebased D. (#34461) + + * Discovery: Move AbstractDisruptionTestCase to file-based discovery. + * Relates #33675 + * Simplify away ClusterDiscoveryConfiguration + +commit af2877cb7b00c2e2892b6d4886151e8a3872a154 +Author: Benjamin Trent +Date: Tue Oct 16 08:55:49 2018 -0500 + + Rollup adding support for date field metrics (#34185) (#34200) + + * Rollup adding support for date field metrics (#34185) + + * Restricting supported metrics for `date` field rollup + + * fixing expected error message for yaml test + + * Addressing PR comments + +commit 950ca3adda36c62e8acc647404d7a94167664fcb +Merge: 5fbead00a3b 230ad5339bb +Author: David Turner +Date: Tue Oct 16 14:41:14 2018 +0100 + + Merge branch 'master' into zen2 + +commit 230ad5339bbb24fe9c50bd5bfa54f2614256fb2a +Author: Vladimir Dolzhenko +Date: Tue Oct 16 15:21:30 2018 +0200 + + HLRC: Add support for XPack Post Start Basic Licence API (#33606) + + + Relates to #29827 + +commit 936faba6c4319ad5972ae974681b5882e3cc8448 +Author: Paul Sanwald +Date: Tue Oct 16 09:02:29 2018 -0400 + + Add delete rollup job support to HL REST Client (#34066) + + Add support for delete rollup job to HL REST Client. + +commit d43a1fac332cd11a05f5108d78949921234a8359 +Author: Simon Willnauer +Date: Tue Oct 16 14:53:07 2018 +0200 + + Lock down Engine.Searcher (#34363) + + `Engine.Searcher` is non-final today which makes it error prone + in the case of wrapping the underlying reader or lucene `IndexSearcher` + like we do in `IndexSearcherWrapper`. Yet, there is no subclass of it yet + that would be dramatic to just drop on the floor. With the start of development + of frozen indices this changed since in #34357 functionality was added to + a subclass which would be dropped if a `IndexSearcherWrapper` is installed on an index. + This change locks down the `Engine.Searcher` to prevent such a functionality trap. + +commit a1ec91395cb48d22e77bc08fce562ab655c4434f +Author: Martijn van Groningen +Date: Tue Oct 16 14:45:46 2018 +0200 + + Changed CCR internal integration tests to use a leader and follower cluster instead of a single cluster (#34344) + + The `AutoFollowTests` needs to restart the clusters between each tests, because + it is using auto follow stats in assertions. Auto follow stats are only reset + by stopping the elected master node. + + Extracted the `testGetOperationsBasedOnGlobalSequenceId()` test to its own test, because it just tests the shard changes api. + + * Renamed AutoFollowTests to AutoFollowIT, because it is an integration test. + Renamed ShardChangesIT to IndexFollowingIT, because shard changes it the name + of an internal api and isn't a good name for an integration test. + + * move creation of NodeConfigurationSource to a seperate method + + * Fixes issues after merge, moved assertSeqNos() and assertSameDocIdsOnShards() methods from ESIntegTestCase to InternalTestCluster, so that ccr tests can use these methods too. + +commit 67e7464601b993bce5540241da186fa1a56553bf +Author: David Turner +Date: Tue Oct 16 12:16:02 2018 +0100 + + Add Punjabi to unsupported Kerberos locales list (#34440) + + Relates #33253 and fixes another test failure similar to #33228. + +commit 145bafea888bdf6741738c2386e2a6bca244cdb9 +Author: Jason Tedor +Date: Tue Oct 16 06:46:06 2018 -0400 + + Add settings assertion in internal index reindexer + + When performing an internal reindex, we add a setting marking the source + as read-only. We also check that this index is not already + read-only. This means that when we add the read-only setting, we expect + that it is already not there. This commit adds an assertion before we + increment the settings version validating that this is indeed the case. + +commit 05911fb49917f3e61c328dcc746172ee759f114e +Author: Jason Tedor +Date: Tue Oct 16 06:33:45 2018 -0400 + + Adjust settings version BWC version after backport + + This commit adjusts the settings version BWC version after backporting + the change to the 6.x branch which currently is versioned as 6.5.0. + +commit 544de13d8e352516e4e1afc0177ff8b1e5580ba5 +Author: Jim Ferenczi +Date: Tue Oct 16 11:31:53 2018 +0100 + + Disallow negative query boost (#34486) + + This change disallows negative query boosts. Negative scores are not allowed in Lucene 8 so + it is easier to just disallow negative boosts entirely. We should also deprecate negative boosts + in 6x in order to ensure that users are aware when they'll upgrade to ES 7. + + Relates #33309 + +commit 4b2052c683b74f66382f13b5c98aaf7750441ffa +Author: Jason Tedor +Date: Tue Oct 16 06:22:20 2018 -0400 + + Introduce index settings version (#34429) + + This commit introduces settings version to index metadata. This value is + monotonically increasing and is updated on settings updates. This will + be useful in cross-cluster replication so that we can request settings + updates from the leader only when there is a settings update. + +commit e0b6721df438b119aa2d9cedfafcdbe16efdec03 +Author: Jason Tedor +Date: Tue Oct 16 06:21:28 2018 -0400 + + Add dedicated test for chain replication (#34497) + + This commit adds a dedicated test that chain replication leader -> + middle -> follow is successful. + +commit 75c973f57e931fd35db0cd1edfb25b696743091a +Author: markharwood +Date: Tue Oct 16 09:10:48 2018 +0100 + + Test fix - GraphExploreResponse HLRC xContent ordering was unreliable(#34473) + + xContent ordering is unreliable when derived from map insertions but the parsed objects’ .equals() methods have the sort logic required to prove connections and vertices are correct. Disabled the xContent equivalence checks. + + Closes #33686 + +commit 2645574a3129cf4c718a83019503c403cadab144 +Author: Alexander Reelsen +Date: Tue Oct 16 09:07:27 2018 +0200 + + Watcher: Remove test-only setting (#34377) + + In 54cb890 a setting for testing only was introduced, that delayed the start up of watcher. With the changes of how is watcher is started/stopped over time, this is not needed anymore. + +commit f7df8718b9b3b3cf5e3d348da60b8dc9dd5e2b4f +Author: Martijn van Groningen +Date: Tue Oct 16 07:44:15 2018 +0200 + + [CCR] Don't fail shard follow tasks in case of a non-retryable error (#34404) + +commit 92b2e1a2091ce30d6267029b7b7b19d4b6690d64 +Author: Daniel Mitterdorfer +Date: Tue Oct 16 06:30:00 2018 +0200 + + Remove lenient boolean handling + + With this commit we remove some leftovers from #26389 which cleaned up + lenient boolean handling. + + Relates #26389 + Relates #22298 + Relates #34467 + +commit f5ef2482df3d14e7e71e2b14bafb4ad8395996fa +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Oct 16 05:12:11 2018 +0100 + + HLRC: Fix the tags to be in sync with docs. (#34501) + + Correct the tags mapping with the documentation. + This was added in #34171 + +commit a4c302067e6e3a2f8dd021440b50f482177fa106 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Oct 16 03:05:46 2018 +0100 + + HLRC: Create/Update role mapping API (#34171) + + We added support for role mapper expression DSL in #33745, + that allows us to build the role mapper expression used in the + role mapping (as rules for determining user roles based on what + the boolean expression resolves to). + + This change now adds support for create/update role mapping + API to the high-level rest client. + +commit 55dee530462cb077395033564b06dd36ac091f05 +Author: Jason Tedor +Date: Mon Oct 15 19:49:58 2018 -0400 + + Do not update number of replicas on no indices (#34481) + + Today when submitting an update settings request to update the number of + replicas with a wildcard that does not match any indices and allow no + indices is set to true, the request ends up being interpreted as + updating the number of replicas for all indices. That is, consider the + following sequence: + + PUT /test-index + { + "settings": { + "index.number_of_replicas": 0 + } + } + + PUT /non-existent-*/_settings?expand_wildcards=open&allow_no_indices=true + { + "settings": { + "index.number_of_replicas": 1 + } + } + + GET /test-index/_settings + + The latter will show that the number of replicas on test-index is now + one. This is surprising, and should be considered a bug. + + The underlying problem here is treating no indices in the underlying + methods used to update the routing table and the metadata as meaning all + indices. This commit takes away this assumption. Tests that relied on + this behavior have been changed to no longer rely on this. + + A test for this situation is added in UpdateNumberOfReplicasIT. + +commit 23ece922c9ea9415bcfb914d8d01846446b33737 +Author: Nik Everett +Date: Mon Oct 15 16:05:14 2018 -0400 + + Core: Remove two methods from AbstractComponent (#34336) + + This removes another two methods from `AbstractComponent`. One isn't + used at all and another is only used in a single class in watcher. I've + moved the method that watcher uses into the single class that uses it. + +commit 0cd03d358140193cc84434d385947e84e0b70607 +Author: Jay Modi +Date: Mon Oct 15 20:52:54 2018 +0100 + + Use RoleRetrievalResult for better caching (#34197) + + Security caches the result of role lookups and negative lookups are + cached indefinitely. In the case of transient failures this leads to a + bad experience as the roles could truly exist. The CompositeRolesStore + needs to know if a failure occurred in one of the roles stores in order + to make the appropriate decision as it relates to caching. In order to + provide this information to the CompositeRolesStore, the return type of + methods to retrieve roles has changed to a new class, + RoleRetrievalResult. This class provides the ability to pass back an + exception to the roles store. This exception does not mean that a + request should be failed but instead serves as a signal to the roles + store that missing roles should not be cached and neither should the + combined role if there are missing roles. + + As part of this, the negative lookup cache was also changed from an + unbounded cache to a cache with a configurable limit. + + Relates #33205 + +commit a6d1cc6ca982ad0eb4b190ae727462b5ea9fcf14 +Author: Nik Everett +Date: Mon Oct 15 15:42:11 2018 -0400 + + Revert "Search: Fix spelling mistake in Javadoc (#34480)" + + This reverts commit 4e1d7baed0a1e2fa1fa17fe4479045d811f1e02e. + +commit 4e1d7baed0a1e2fa1fa17fe4479045d811f1e02e +Author: fonxian +Date: Tue Oct 16 03:38:37 2018 +0800 + + Search: Fix spelling mistake in Javadoc (#34480) + + "iff" -> "if". + +commit f5641e61a25fe90804ab5992f24f3126e1e9d553 +Author: Nikolay Vasiliev +Date: Mon Oct 15 21:30:48 2018 +0200 + + Docs: improve formatting of Query String Query doc page (#34432) + + Merge two tables. + +commit 26f1d7fc947ff09ce70fd98748f28f5cabda72c3 +Author: Ryan Ernst +Date: Mon Oct 15 12:18:18 2018 -0700 + + Tests: Handle epoch date formatters edge cases (#34437) + + This commit handles cases testing withLocale and withZone when the zone + and locale in question is the same as the special base case. This can + happen sometimes since the locale and zoneids are randomized. + +commit 79c735a04d6da7a087b83e6678fcac85d684e57c +Author: Nik Everett +Date: Mon Oct 15 15:10:43 2018 -0400 + + Test: Fix running with external cluster + + Back in #32983 I broke running the integ-test-zip tests against an + external cluster by adding a test that reads the contents of the log + file. This fixes running against an external cluster by explicitly + skipping that test if running against an external cluster. + +commit 67577fca5607dc0cefb20aaf475f66ede8a5a6fe +Author: Jim Ferenczi +Date: Mon Oct 15 19:33:52 2018 +0100 + + Fix handling of empty keyword in terms aggregation (#34457) + + Empty values on keyword fields are filtered by the `map` execution mode + of the `terms` aggregation. This commit restores them as valid buckets. + + Closes #34434 + +commit 8e133ab4519fdf6dc754bf35efe111dd52c83c0d +Author: Ioannis Kakavas +Date: Mon Oct 15 20:03:47 2018 +0300 + + [DOCS] Fix tag in SecurityDocumentationIT + +commit fb3123618812b6329536a8e44413f3502e8f9797 +Author: Chris Roberson +Date: Mon Oct 15 12:37:52 2018 -0400 + + [Monitoring] Add additional necessary mappings for apm-server (#34392) + + * Add additional necessary mappings for apm-server + + * Add open handles for beats + + * Add mappings missing for es + +commit ebca27371c20e9ca26f4e75c4228d27316166345 +Author: Armin Braun +Date: Mon Oct 15 17:28:05 2018 +0100 + + SCRIPTING: Move Aggregation Script Context to its own class (#33820) + + * SCRIPTING: Move Aggregation Script Context to its own class + +commit 511526250b1893aeb5c026c100026ebde751baff +Author: Armin Braun +Date: Mon Oct 15 17:26:08 2018 +0100 + + MINOR: Remove Deadcode in ExpressionTermSetQuery (#34442) + +commit 55eaf7a3ff83d032fd4f2263e144113909d1b2bd +Author: Ioannis Kakavas +Date: Mon Oct 15 17:20:34 2018 +0100 + + HLRC: Get SSL Certificates API (#34135) + + This change adds support for the get SSL certificate API to + the high level rest client. + +commit 0b42eda0e360d17975460a0d1efd0c6e6b494ca5 +Merge: 9ad2a7fa77e 51eca14288e +Author: Colin Goodheart-Smithe +Date: Mon Oct 15 16:03:37 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 9bb620eecefb40e2bad2ce5014a88a1e77ed7269 +Author: David Turner +Date: Mon Oct 15 13:18:00 2018 +0100 + + Mute PartitionedRoutingIT#testShrinking on Windows + +commit e3a1981a574b696b95c52e61ebfc64db52d593de +Author: Andrey Ershov +Date: Mon Oct 15 14:08:04 2018 +0200 + + Mute testToQuery test + +commit 51eca14288ec76a6533dca5ed5dedd5cffedf33b +Author: Martijn van Groningen +Date: Mon Oct 15 10:24:28 2018 +0200 + + [TEST] Make sure there are shards started so that `ESIntegTestCase#assertSameDocIdsOnShards()` does not fail with shard not found. + +commit 74dc2da8731e2e666beafeb32a39e50ac47395be +Author: Martijn van Groningen +Date: Mon Oct 15 08:09:00 2018 +0100 + + Change shard changes api's threadpool from get to search (#34421) + +commit 305b5845eb49fabf4988e7be9fe1a20d7e4a96d4 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Mon Oct 15 07:54:18 2018 +0200 + + Update TESTING.asciidoc title (#34401) + + Display `Testing` as h1 + +commit 72d818c304f256e8ec07802d802b4dc8dcb9b0b2 +Author: Ryan Ernst +Date: Sun Oct 14 23:54:49 2018 +0100 + + Tests: Fix DateFormatter equals tests with locale (#34435) + + This commit removes randomization of locale for DateFormatter equals + tests, instead using explicit locales. The test framework already + randomizes locales, so the random choice of the second locale can + sometimes be equal to the already chosen locale. Randomization also does + not provide any extra protection, as the equality of DateFormatter does + not implement equality of the locales itself. + + closes #34337 + +commit a6e8fb33c08deec0c41799768a2035edff8d7a0b +Author: Ryan Ernst +Date: Sun Oct 14 21:32:36 2018 +0100 + + Docs: Remove unnecessary qualifier from wildcard import note (#34419) + + This commit removes the suggestions to send a PR with instructions for + more IDEs on disabling wildcard imports. This is not explicitly needed + both because the main IDEs instructions already exist, and any part of + the docs are welcome for PRs to add more clarity. + + closes #34415 + +commit 5fbead00a3bf8fd5ec20502e4cbcc5a124763546 +Author: Yannick Welsch +Date: Sun Oct 14 20:55:04 2018 +0100 + + Zen2: Add infrastructure for integration tests (#34365) + + Adds the infrastructure to run integration tests against Zen2. + +commit 429c29e833c7ecdf7caafeb38fb071dc0ddc82de +Author: Nhat Nguyen +Date: Sat Oct 13 21:05:33 2018 -0400 + + CCR/TEST: AwaitsFix testFailOverOnFollower + + Tracked at #34412 + +commit 8b9fa55c93cf8bcd859daf285e0ec3947d1f51f3 +Author: David Turner +Date: Sat Oct 13 14:24:15 2018 +0100 + + Add storage-layer disruptions to CoordinatorTests (#34347) + + Today we assume the storage layer operates perfectly in CoordinatorTests, which + means we are not testing that the system's invariants are preserved if the + storage layer fails for some reason. This change injects (rare) storage-layer + failures during the safety phase to cover these cases. + +commit a7c4dbbb0cdd8c91cab94c816d6d11a1ad615ca9 +Author: Jack Conradson +Date: Fri Oct 12 15:34:48 2018 -0700 + + [Painless] Add a Map for java names to classes for use in the custom classloader (#34424) + + This fixes a bug that wasn't including the class used for a static import + where only the static import is whitelisted. + +commit 100a18bd8dbf79261fbdf2d3ff7fe4be5d3b6b6f +Author: Nhat Nguyen +Date: Fri Oct 12 15:11:55 2018 -0400 + + TEST: Fix indentation in FullClusterRestartIT (#34420) + +commit 080ddd5d9c31c01eaef5aaee24a65dcf473178f0 +Author: Ioannis Kakavas +Date: Fri Oct 12 17:09:14 2018 +0100 + + [WIP] Ingest Attachement: Upgrade tika to v1.19.1 (#33896) + + Upgrades Tika to 1.19.1 and relevant transitive dependencies + + Resolves: #31456, #31305 + +commit 7ba320d5b22e515c28389d2d22db10553250798f +Author: Armin Braun +Date: Fri Oct 12 16:13:28 2018 +0100 + + NETWORKING: Upgrade Netty to 4.1.30 (#34417) + + * closes #34411 + +commit 8d83688328f17c05ad7a7404a6e4319b9428e477 +Author: Tim Vernum +Date: Fri Oct 12 14:58:04 2018 +0100 + + Allow an AuthenticationResult to return metadata (#34382) + + PR #34290 made it impossible to use thread-context values to pass + authentication metadata out of a realm. The SAML realm used this + technique to allow the SamlAuthenticateAction to process the parsed + SAML token, and apply them to the access token that was generated. + + This new method adds metadata to the AuthenticationResult itself, and + then the authentication service makes this result available on the + thread context. + + Closes: #34332 + +commit 21c759af0ed5996fbb413f6aa64598bba96a238c +Author: David Roberts +Date: Fri Oct 12 07:56:35 2018 +0100 + + [ML] Add an ingest pipeline definition to structure finder (#34350) + + The ingest pipeline that is produced is very simple. It + contains a grok processor if the format is semi-structured + text, a date processor if the format contains a timestamp, + and a remove processor if required to remove the interim + timestamp field parsed out of semi-structured text. + + Eventually the UI should offer the option to customize the + pipeline with additional processors to perform other data + preparation steps before ingesting data to an index. + +commit d98199df14c0986695e84de3df2daad464444d5b +Author: David Turner +Date: Thu Oct 11 23:24:08 2018 +0100 + + Extend duration of fixLag() (#34364) + + Today, fixLag() waits for a new cluster state to be committed. However, it does + not account for the fact that a term bump may occur, requiring a new election + to take place after the cluster state is committed. This change fixes this. + +commit 7352f0da60a2230dacc8fc13ae7690981f856926 +Author: David Turner +Date: Thu Oct 11 15:33:09 2018 +0100 + + Handle pre-6.x time fields (#34373) + + In ccb9ab5717d85c6786b081f197d67ac7dde4d317 we changed how we deal with time + fields to support the `DateTime`-format fields added in 6.0, but dropped + support for pre-6.x `Long`-format fields. This change reinstates this support + for cases where pre-6.x data is made available to ML (e.g. in a mixed-version + CCS setup or after an upgrade). + +commit a32e303b0c43e69d5a1d4feaebcac4e5b26d4c13 +Author: David Turner +Date: Thu Oct 11 15:31:08 2018 +0100 + + Account for election duration (#34362) + + Today we may schedule two elections very close together, which can cause the + first election to fail even if there are no other nodes. This change adds a + delay in between subsequent elections on the same node, effectively allowing + time for each election to complete before scheduling the next one. + +commit 6d99d7dafce78a8d6c8cb48bcfeced069e48351b +Author: Jay Modi +Date: Thu Oct 11 15:24:38 2018 +0100 + + ListenableFuture should preserve ThreadContext (#34394) + + ListenableFuture may run a listener on the same thread that called the + addListener method or it may execute on another thread after the future + has completed. Whenever the ListenableFuture stores the listener for + execution later, it should preserve the thread context which is what + this change does. + +commit 7bc11a80993c3cdf1ae02094544734f6c8598678 +Author: Nhat Nguyen +Date: Wed Oct 10 15:48:22 2018 -0400 + + Unmute testFollowIndexAndCloseNode + + This issue was resolved by #34288. + + Closes #33337 + Relates #34288 + +commit 33791ac27c7a9591ea2d873fa148e2427cd06b75 +Author: Nhat Nguyen +Date: Wed Oct 10 15:39:57 2018 -0400 + + CCR: Following primary should process operations once (#34288) + + Today we rewrite the operations from the leader with the term of the + following primary because the follower should own its history. The + problem is that a newly promoted primary may re-assign its term to + operations which were replicated to replicas before by the previous + primary. If this happens, some operations with the same seq_no may be + assigned different terms. This is not good for the future optimistic + locking using a combination of seqno and term. + + This change ensures that the primary of a follower only processes an + operation if that operation was not processed before. The skipped + operations are guaranteed to be delivered to replicas via either + primary-replica resync or peer-recovery. However, the primary must not + acknowledge until the global checkpoint is at least the highest seqno of + all skipped ops (i.e., they all have been processed on every replica). + + Relates #31751 + Relates #31113 + +commit 4270085360302dc26700f228bf083f1594f81ade +Author: Jack Conradson +Date: Wed Oct 10 10:00:45 2018 -0700 + + [Painless] Allow statically imported methods without whitelist class. (#34370) + + This removes the extraneous check to see if the class for a statically imported + method is already whitelisted, so a statically imported method can be whitelisted + independently. + +commit 34b935ae57bd513db77327074534cdf8f5dd02e7 +Author: Simon Willnauer +Date: Wed Oct 10 17:28:07 2018 +0100 + + Improve `getRestHandlerWrapper` JavaDocs (#34376) + + Questions on how to work with `ActionPlugin#getRestHandlerWrapper()` + come up in discuss forums all the time. This change adds an example + to the javadoc how this method should/could be used. + +commit 4b7257d971f18627467a1e9156f94dfe9eabe6e2 +Author: Ryan Ernst +Date: Tue Oct 9 17:07:51 2018 -0400 + + Build: Pass offline flag through to bwc build (#34367) + + This commit makes the bwc build aware of the offline flag passed to the outer build. + + closes #34361 + +commit d7a94fb6aa1f85709a2a03ca30099fa8fbb04395 +Author: Andrei Stefan +Date: Tue Oct 9 00:20:18 2018 +0300 + + SQL: Functions enhancements (OCTET_LENGTH function, order functions alphabetically, RANDOM function docs) (#34101) + + * New OCTET_LENGTH function + * Changed the way the FunctionRegistry stores functions, considering the alphabetic ordering by name + * Added documentation for the RANDOM function + +commit 9ad2a7fa77e877bee18a832624486740b7e0d37e +Author: Lee Hinman +Date: Mon Oct 8 11:25:18 2018 -0600 + + Fix expected next step being incorrect when executing async action (#34313) + + This fixes an issue where an incorrect expected next step is used when checking + to execute `AsyncActionStep`s after a cluster state step. + + It fixes this scenario: + + - `ExecuteStepsUpdateTask` executes a `ClusterStateWaitStep` or + `ClusterStateActionStep` successfully + - The next step is also a `ClusterStateWaitStep`, so it loops + - The `ClusterStateWaitStep` has a next stepkey (which gets set to the + `nextStepKey` in the code) + - The `ClusterStateWaitStep` fails the condition, meaning that it will have to + wait longer + - The `nextStepKey` is now incorrect though, because we did not advance the + index's step, and it's not `null` (which is another safe value if there is no + step after the `ClusterStateWaitStep`) + + This fixes the problem by resetting the nextStepKey to null if the condition is + not met, since we are not going to advance the step metadata in this + case (thereby skipping the `maybeRunAsyncAction` invocation). + + This commit also tightens up and enhances much of the ILM logging. A lot of + logging was missing the index name (making it hard to debug in the presence of + multiple indices) and a lot was using the wrong logging level (DEBUG is now + actually readable without being a wall of text). + + Resolves #34297 + +commit 52a3a1955157c4179c743b8d8dd9cd8571982fdd +Author: David Turner +Date: Mon Oct 8 15:56:48 2018 +0100 + + Add low-level bootstrap implementation (#34345) + + Today we inject the initial configuration of the cluster (i.e. the set of + voting nodes) at startup. In reality we must support injecting the initial + configuration after startup too. This commit adds low-level support for doing + so as safely as possible. + +commit 9e522d5d97b7a6a9fa40a0b4304cd6c7c7f88312 +Author: Yannick Welsch +Date: Mon Oct 8 15:39:00 2018 +0200 + + Use RUNTIME_JAVA_HOME for micro benchmarks (#34353) + + We don't honour RUNTIME_JAVA_HOME for running the benchmarks. It's useful to run some of the + benchmarks against older JDKs, though. + +commit 49cbcaff4f33637ec09c5d5b48267dccf199987f +Author: Yannick Welsch +Date: Mon Oct 8 15:35:50 2018 +0200 + + Allow excluding folder names when scanning for dangling indices (#34349) + + ES is scanning for dangling indices on every cluster state update. For this, it lists the subfolders of + the indices directory to determine which extra index directories exist on the node where there's no + corresponding index in the cluster state. These are potential targets for dangling index import. On + certain machine types, and with large number of indices, this subfolder listing can be horribly slow. + This means that every cluster state update will be slowed down by potentially hundreds of + milliseconds. One of the reasons for this poor performance is that Files.isDirectory() is a relatively + expensive call on some OS and JDK versions. There is no need though to do all these isDirectory + calls for folders which we know we are going to discard anyhow in the next step of the dangling + indices logic. This commit allows adding an exclusion predicate to the availableIndexFolders + methods which can dramatically speed up this method when scanning for dangling indices. + +commit 268e134121ddd27456f0039d1bb51dfc266418da +Author: Martijn van Groningen +Date: Mon Oct 8 15:05:50 2018 +0200 + + renamed test class + +commit ac99d1d66df475a3bb38bfa2b0f3d736063ad6ec +Author: David Turner +Date: Mon Oct 8 11:33:25 2018 +0100 + + Fix bugs in fixLag() (#34346) + + The hack to work around lag detection had some issues: + - it always called runFor(), even if no lag was detected + - it looked at the last-accepted state not the last-applied state, so missed + some lag situations. + + This fixes these issues. + +commit 6f32f7138db47f503974bd7ce71f32449289a3c7 +Author: Benjamin Trent +Date: Sat Oct 6 07:25:15 2018 -0700 + + HLRC: Fixing bug when getting a missing pipeline (#34286) + + closes #34119 + +commit 06993e0c356cd549fdbea759e214aabd289510a6 +Author: Nik Everett +Date: Sat Oct 6 09:54:08 2018 -0400 + + Logging: Make ESLoggerFactory package private (#34199) + + Since all calls to `ESLoggerFactory` outside of the logging package were + deprecated, it seemed like it'd simplify things to migrate all of the + deprecated calls and declare `ESLoggerFactory` to be package private. + This does that. + +commit c6c83d19f7ece7affd2f24f1c16ee1a4721106e0 +Author: Martijn van Groningen +Date: Sat Oct 6 07:53:37 2018 -0400 + + [CCR] Clear fetch exceptions if an empty but successful shard changes response returns (#34256) + + Also fixed ShardFollowNodeTaskTests to not return ops when responseSize + is empty. Otherwise ops are returned when no ops are expected to be returned. + + Co-authored-by: Jason Tedor + +commit 899e48395ba1f0a313e431388216cbaea0078caa +Author: Martijn van Groningen +Date: Sat Oct 6 07:38:28 2018 -0400 + + [CCR] Change unfollow API's privilege scheme. (#34175) + + Unfollow should be allowed / disallowed on a per index level instead of + cluster level. + + Also renamed `create_follow_index` index privilege to + `manage_follow_index` privilege and include unfollow and close APIs. + +commit 03da4f6c518af5e2678903d25dc7362ba46b50dd +Author: David Turner +Date: Sat Oct 6 07:22:04 2018 +0100 + + Gather votes from all nodes (#34335) + + Today we accept that some nodes may vote for the wrong master in an election. + This is mostly fine because they do end up joining the correct master in the + end, but the lack of a vote from every follower may prevent a future desirable + reconfiguration from taking place. + + The solution is to hold another election in a yet-higher term in order to + collect a complete set of votes. Elections are somewhat disruptive so we should + think carefully about when this election should take place. One option is to + wait as late as possible (on the grounds that it might not ever be necessary). + This unfortunately makes it harder to predict how an + apparently-smoothly-running cluster will react to nodes leaving and joining. + Instead we prefer to perform the election as soon as possible in the leader's + term, adding "votes from all followers" to the invariants that we expect to + hold in a stable cluster. The start of a leader's term is already a somewhat + disrupted time for the cluster, so performing another election at this point + does not materially change the cluster's behaviour. + + This change implements the logic needed to trigger a new election in order to + satisfy this extra stabilisation condition. + +commit d905cc8fc892c0a9586525f567dbf3468ac400e8 +Author: Nik Everett +Date: Fri Oct 5 23:52:39 2018 -0400 + + LLRC: Test for warnings behavior (#34143) + + Add tests for the Low Level REST Client's strict deprecation handling. + + Relates to #33708 + +commit 36557469f679bfcf2399cd519c541e23bdf13b48 +Author: Jim Ferenczi +Date: Fri Oct 5 19:46:20 2018 +0200 + + [DOCS] Removes beta label from composite aggregation (#34329) + +commit 6e28c8f1c454c2b09a7820407001a72bcc76f06e +Author: Jim Ferenczi +Date: Fri Oct 5 19:45:23 2018 +0200 + + [DOCS] Remove experimental label from term_set query (#34328) + +commit cfe8eab455e7da49e1d6abb10a0499ef704bac8c +Author: Jim Ferenczi +Date: Fri Oct 5 19:44:25 2018 +0200 + + [DOCS] Removes beta label from index sorting (#34327) + +commit 1bd8ff520b174c0ef3cdcd1404f2b5dd44433da8 +Author: Nathan Delhaye +Date: Fri Oct 5 19:17:55 2018 +0200 + + Typo in x-pack template for thread_pool.management (#34224) + + Typo in x-pack template for the node_stats.thread_pool.management + +commit c031322c18496883d45d48d9ef15bec277632122 +Author: Alpar Torok +Date: Fri Oct 5 18:57:23 2018 +0300 + + [Build] randomizedtesting: Allow property values to be closures (#34319) + + Makes it easier to have property values that are evaluated at build time + rather than configuration time in a Gradle idiomatic way. + +commit 6f5daadb3c4a51a00d8112b5d8c99d10c69fb981 +Author: Benjamin Trent +Date: Fri Oct 5 08:45:46 2018 -0700 + + Feature/hlrc ml docs cleanup (#34316) + + * HLRC: ML Add preview datafeed api + + * Changing deprecation handling for parser + + * Removing some duplication in docs, will address other APIs in another PR + + * HLRC: ML Cleanup docs + + * updating get datafeed stats docs + +commit a9fa5f2b33513805b7aa4e554efd543c6815891b +Author: Nik Everett +Date: Fri Oct 5 11:41:03 2018 -0400 + + Docs: DRY up CRUD docs (#34203) + + This further applies the pattern set in #34125 to reduce copy-and-paste + in the single document CRUD portion of the High Level REST Client docs. + It also adds line wraps to snippets that are too wide to fit into the box + when rendered in the docs, following up on the work started in #34163. + +commit e95dc5474fc97b8380ed34c867558663f0be94ab +Author: amoreauCoveo +Date: Fri Oct 5 10:08:03 2018 -0400 + + Minor corrections in geo-queries.asciidoc (#34314) + +commit a9daa5cb903b6adb64ecd5b4268c0c02c5751940 +Author: Jim Ferenczi +Date: Fri Oct 5 15:42:00 2018 +0200 + + [DOCS] Remove beta label from normalizers (#34326) + +commit 7d826916b9150e0555b0a35b20f05f005642cdd0 +Author: Daniel Mitterdorfer +Date: Fri Oct 5 15:39:08 2018 +0200 + + Adjust size of BigArrays in circuit breaker test + + With this commit we restore the previous behavior in + `BigArraysTests#testMaxSizeExceededOnResize` but lower the sizes that + are tested to the range between 256 bytes to 16 kB so the test does not + produce a whole lot of garbage. + + The previous attempt to reduce the amount of garbage produced by that + test was to properly size the array initially but it failed to account + for object alignment which lead to test failures in some cases. While it + would be possible to account for object alignment, we would need to open + up BigArrays or directly use the underlying Lucene API which would + require us to allocate an array upfront only to find its size (incl. + object alignment). + + Instead we have fixed this issue by conservatively sizing the array + initially (so the initial allocation will never trip the circuit + breaker) and reduce garbage by reducing the circuit breaker's upper + bound as described previously. + + Closes #33750 + Relates #34325 + +commit 5c7b52e9307376f43d10cef14ccd5223367f4e83 +Author: Jim Ferenczi +Date: Fri Oct 5 13:07:08 2018 +0200 + + Adapt bwc version after backport + + Relates #33587 + +commit 7d57bdb3a0dd2bbb82701225c9dbfd69a5011814 +Author: Jason Tedor +Date: Fri Oct 5 06:38:20 2018 -0400 + + Follow stats structure (#34301) + + This commit modifies the follow stats API response structure to more + clearly highlight meaning of the higher level fields. In particular, + previously the response had a top-level key for each index. Instead, we + nest the indices under an "indices" field which is now an array. The + values in this array are objects containing two fields: "index" which is + the name of the follower index, and "shards" which is an array where + each value in the array is the follower stats for that shard. That is, + we have gone from: + + { + "bar": [ + { + "shard_id": 0... + }... + ]... + } + + to + + { + "indices": [ + { + "index": "bar", + "shards": [ + { + "shard_id": 0... + }... + ] + }... + } + +commit 7478167d603e78d00fad27c9a3855b23da919f78 +Author: Jason Tedor +Date: Fri Oct 5 06:25:24 2018 -0400 + + Rename CCR stats implementation (#34300) + + In the CCR docs we want to refer to the endpoint that returns following + stats as the follow stats API. This commit renames the internal + implementation of this endpoint to reflect this usage. + +commit daf88335d75399f31e315dccd7448890dc60fb00 +Author: eray +Date: Fri Oct 5 13:02:47 2018 +0300 + + Add max_children limit to nested sort (#33587) + + Add an option to `nested` sort to limit the number of children to visit when picking the sort value + of the root document. + + Closes #33592 + +commit 732ab06ee498f074872ae4898e7402c07cfb3eaa +Author: Armin Braun +Date: Fri Oct 5 10:53:03 2018 +0200 + + MINOR: Remove Dead Code from Netty4Transport (#34134) + + * None of these methods are used + +commit 7df842c78d3e14b6f920c8506399d9ec674c7260 +Author: Alpar Torok +Date: Fri Oct 5 10:51:04 2018 +0300 + + Rename clsuterformation -> testclusters (#34299) + +commit 0065e72be60dbc4eeb7400df6909b40491db6302 +Author: Alpar Torok +Date: Fri Oct 5 10:47:15 2018 +0300 + + [Build] make sure there are no duplicate classes in third party audit (#34213) + + * make sure there are no duplicate classes in third party audit + +commit 76310eb3f16110325dc7798a7a1db907f019c8e8 +Author: Alpar Torok +Date: Fri Oct 5 10:46:00 2018 +0300 + + BWC Build: Read CI properties to determine java version (#34295) + +commit f420eebc735efc91eb0583380f851072d1d41aec +Author: Ioannis Kakavas +Date: Fri Oct 5 10:07:50 2018 +0300 + + [DOCS] Fix typo and add [float] + +commit 29d7d1d50349b4edacd5ed5b7531c577e929a113 +Author: David Turner +Date: Fri Oct 5 07:57:03 2018 +0100 + + Minor housekeeping of tests (#34315) + + From experience with #34257, here are a few things that help with analysing + logs from test runs. Also we prevent trying to stabilise a cluster with raised + delay variability, because lowering the delay variability requires time to + allow all the extra-varied-scheduled tasks to work their way out of the system. + +commit 63dbd1dce0258d6872e3fd5bad3cadad44b820fb +Author: Tim Vernum +Date: Fri Oct 5 12:10:42 2018 +1000 + + Allow User/Password realms to disable authc (#34033) + + The "lookupUser" method on a realm facilitates the "run-as" and + "authorization_realms" features. + This commit allows a realm to be used for "lookup only", in which + case the "authenticate" method (and associated token methods) are + disabled. + It does this through the introduction of a new + "authentication.enabled" setting, which defaults to true. + +commit 66089925231abf5dad4901502977ff0d2dd0cd49 +Author: Tim Vernum +Date: Fri Oct 5 12:09:36 2018 +1000 + + Enable security automaton caching (#34028) + + Building automatons can be costly. For the most part we cache things + that use automatons so the cost is limited. + However: + - We don't (currently) do that everywhere (e.g. we don't cache role + mappings) + - It is sometimes necessary to clear some of those caches which can + cause significant CPU overhead and processing delays. + + This commit introduces a new cache in the Automatons class to avoid + unnecesarily recomputing automatons. + +commit 1bb2a1502d705ea966b2a0d3d05025907f0ff28b +Author: Tim Vernum +Date: Fri Oct 5 12:08:21 2018 +1000 + + Preserve thread context during authentication. (#34290) + + There may be values in the thread context that ought to be preseved + for later use, even if one or more realms perform asynchronous + authentication. + + This commit changes the AuthenticationService to wrap the potentially + asynchronous calls in a ContextPreservingActionListener that retains + the original thread context for the authentication. + +commit 4dacfa95d2a2e88cec7e9d9d80af5a39542397a8 +Author: Dimitris Athanasiou +Date: Fri Oct 5 02:41:28 2018 +0300 + + [ML] Allow asynchronous job deletion (#34058) + + This changes the delete job API by adding + the choice to delete a job asynchronously. + The commit adds a `wait_for_completion` parameter + to the delete job request. When set to `false`, + the action returns immediately and the response + contains the task id. + + This also changes the handling of subsequent + delete requests for a job that is already being + deleted. It now uses the task framework to check + if the job is being deleted instead of the cluster + state. This is a beneficial for it is going to also + be working once the job configs are moved out of the + cluster state and into an index. Also, force delete + requests that are waiting for the job to be deleted + will not proceed with the deletion if the first task + fails. This will prevent overloading the cluster. Instead, + the failure is communicated better via notifications + so that the user may retry. + + Finally, this makes the `deleting` property of the job + visible (also it was renamed from `deleted`). This allows + a client to render a deleting job differently. + + Closes #32836 + +commit 026488bcbfd9a2f3a45ae48e8c0708f3d113434b +Author: Benjamin Trent +Date: Thu Oct 4 14:43:06 2018 -0700 + + HLRC: ML Adding get datafeed stats API (#34271) + + * HLRC: ML Adding get datafeed stats API + + * addressing PR comments + + * fixing field exclusion filter + + * removing unnecessary whitespace + +commit a6c5b6807fc02a46c78d29c90486b1ef72870746 +Author: Julie Tibshirani +Date: Thu Oct 4 23:30:20 2018 +0200 + + Small fixes to the HLRC watcher documentation. (#34306) + + * Make sure to use the new docs style for 'ack watch'. + * Replace 'document' with 'watcher' in page names. + +commit 09aaed4fe49ec662941750de90fd51fcf90e5559 +Author: Nik Everett +Date: Thu Oct 4 14:42:37 2018 -0400 + + Tasks: Document that status is not semvered (#34270) + + The `status` part of the tasks API reflects the internal status of a + running task. In general, we do not make backwards breaking changes to + the `status` but because it is internal we reserve the right to do so. I + suspect we will very rarely excercise that right but it is important + that we have it so we're not boxed into any particular implementation + for a request. + + In some sense this is policy making by documentation change. In another + it is clarification of the way we've always thought of this field. + + I also reflect the documentation change into the Javadoc in a few + places. There I acknowledge Kibana's "special relationship" with + Elasticsearch. Kibana parses `_reindex`'s `status` field and, because + we're friends with those folks, we should talk to them before we make + backwards breaking changes to it. We *want* to be friends with everyone + but there is only so much time in the day and we don't *want* to make + backwards breaking fields to `status` at all anyway. So we hope that + breaking changes documentation should be enough for other folks. + + Relates to #34245. + +commit b32abcbd009ada6e3486e4776e9e4ea3c6cc8c23 +Author: Yannick Welsch +Date: Thu Oct 4 20:33:28 2018 +0200 + + Zen2: Add Cluster State Applier (#34257) + + Adds the cluster state applier to Coordinator, and adds tests for cluster state acking. + +commit 2dd058d60786f4ead88883f72200a273432089e1 +Author: Benjamin Trent +Date: Thu Oct 4 11:28:44 2018 -0700 + + HLRC: ML Add preview datafeed api (#34284) + + * HLRC: ML Add preview datafeed api + + * Changing deprecation handling for parser + + * Removing some duplication in docs, will address other APIs in another PR + +commit dcfe64e0e451925592b2608b88d7e882d01eb3be +Author: Vladimir Dolzhenko +Date: Thu Oct 4 16:31:56 2018 +0200 + + [CI] Fix bogus ScheduleWithFixedDelayTests.testRunnableRunsAtMostOnceAfterCancellation + + Closes #34004 + +commit 13d89295c829b6dfc3b70620ccb00422a6c4f863 +Author: Gordon Brown +Date: Thu Oct 4 08:21:55 2018 -0600 + + Provide useful error when a policy doesn't exist (#34206) + + When an index is configured to use a lifecycle policy that does not + exist, this will now be noted in the step_info for that policy. + +commit f1f0687b57812f0a3f997d0111145ad2b034246a +Author: iverase +Date: Thu Oct 4 16:11:43 2018 +0200 + + Fix error in documentation for activete watch + +commit 3ccfc3de58f87249bd811432ec60f78dfd70d371 +Author: Armin Braun +Date: Thu Oct 4 16:03:57 2018 +0200 + + SCRIPTING: Terms set query expression (#33856) + + * SCRIPTING: Add Expr. Compile for TermSetQuery Ctx. + + * Follow up to #33602 adding the ability to compile TermsSetQuery + scripts with the expressions engine in the same way we support + SearchScript in Expressions + * Duplicated the code here for now to make the change less complex, + the only difference to SearchScript is that `_score` and `_value` are not handled for TermsSetQuery + * remove redundant check + +commit ab8a5563f2d5f580e62fbd228c1de7c0d590662e +Author: Nik Everett +Date: Thu Oct 4 09:18:04 2018 -0400 + + Logging: Drop remaining Settings log ctor (#34149) + + Drops the last logging constructor that takes `Settings` because it is + no longer needed. + + Watcher goes through a lot of effort to pass `Settings` to `Logger` + constructors and dropping `Settings` from all of those calls allowed us + to remove quite a bit of log-based ceremony from watcher. + +commit ef5007b6d89fad848e8c0a706fdfda78f954c459 +Author: David Kyle +Date: Thu Oct 4 13:16:14 2018 +0100 + + [ML] Remove unused last_data_time member from Job (#34262) + +commit dc2cf28fde6a7ec2576cf526eb8e1a238fad2f3e +Author: Nik Everett +Date: Thu Oct 4 08:03:38 2018 -0400 + + Docs: Allow skipping response assertions (#34240) + + We generate tests from our documentation, including assertions about the + responses returned by a particular API. But sometimes we *can't* assert + that the response is correct because of some defficiency in our tooling. + Previously we marked the response `// NOTCONSOLE` to skip it, but this + is kind of odd because `// NOTCONSOLE` is really to mark snippets that + are json but aren't requests or responses. This introduces a new + construct to skip response assertions: + ``` + // TESTRESPONSE[skip:reason we skipped this] + ``` + +commit 3ccb7af56a127371807714d0b869c1952a7685ce +Author: Ignacio Vera +Date: Thu Oct 4 12:49:52 2018 +0200 + + HLRC: Add activate watch action (#33988) + + * HLRC: Add activate watcher action + + Adds activate watch action to the high level rest client. + + Relates #29827 + +commit f817bc57d370cde59938ac7686a2a69d64c8fcef +Author: Albert Zaharovits +Date: Thu Oct 4 11:19:15 2018 +0300 + + [Security] Multi Index Expression alias wildcard exclusion (#34144) + + The Security plugin authorizes actions on indices. Authorization + happens on a per index/alias basis. Therefore a request with a + Multi Index Expression (containing wildcards) has to be + first evaluated in the authorization layer, before the request is + handled. For authorization purposes, wildcards in expressions will + only be expanded to indices/aliases that are visible by the authenticated + user. However, this "constrained" evaluation has to be compatible with + the expression evaluation that a cluster without the Security plugin + would do. Therefore any change in the evaluation logic + in any of these sites has to be mirrored in the other site. + + This commit mirrors the changes in core from #33518 that allowed + for Multi Index Expression in the Get Alias API, loosely speaking. + +commit 577261ee578607349c29f7d0920a7b40718412ff +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Thu Oct 4 09:08:21 2018 +0100 + + [ML] Label anomalies with multi_bucket_impact (#34233) + + * [ML] Label anomalies with multi_bucket_impact + + Add the multi_bucket_impact field to record results. + +commit 74993fe1519b11ae45666ab7af31d1303a7633a5 +Author: Ioannis Kakavas +Date: Thu Oct 4 10:53:55 2018 +0300 + + Document smtp.ssl.trust configuration option (#34275) + + This adds documentation for `smtp.ssl.trust` that was reintroduced + in #31684 in 6.3.2. + + Resolves #32936 + +commit 2c82b80b8529a1b855c2916a83532c32bf65a313 +Author: Ioannis Kakavas +Date: Thu Oct 4 10:51:58 2018 +0300 + + Support PKCS#11 tokens as keystores and truststores (#34063) + + This enables Elasticsearch to use the JVM-wide configured + PKCS#11 token as a keystore or a truststore for its TLS configuration. + The JVM is assumed to be configured accordingly with the appropriate + Security Provider implementation that supports PKCS#11 tokens. + For the PKCS#11 token to be used as a keystore or a truststore for an + SSLConfiguration, the .keystore.type or .truststore.type must be + explicitly set to pkcs11 in the configuration. + The fact that the PKCS#11 token configuration is JVM wide implies that + there is only one available keystore and truststore that can be used by TLS + configurations in Elasticsearch. + The PIN for the PKCS#11 token can be set as a truststore parameter in + Elasticsearch or as a JVM parameter ( -Djavax.net.ssl.trustStorePassword). + + The basic goal of enabling PKCS#11 token support is to allow PKCS#11-NSS in + FIPS mode to be used as a FIPS 140-2 enabled Security Provider. + +commit c6b0f084726a002bfa676fb91c688251f078b6ce +Author: David Turner +Date: Thu Oct 4 07:40:26 2018 +0100 + + Add safety phase to CoordinatorTests (#34241) + + Today's CoordinatorTests have a limited amount of randomisation in how things + are scheduled. However, to be fully confident in Zen2's liveness we require the + system to stabilise after any permitted sequence of events. We can achieve + this by running the system in a much more random fashion for a while, with much + larger variation in when things are scheduled (simulating GC pressure and + network disruption) and then continuing to assert that the system stabilises as + we expect. When running randomly, we do not expect to make significant progress + and merely verify that no safety property is violated. + + This change introduces the runRandomly() test method which implements this + idea. It also fixes a handful of liveness bugs that this first version of + runRandomly() exposed. + +commit e8b986cc37eaf54a9717ef0687daead670eba12c +Author: Jim Ferenczi +Date: Thu Oct 4 07:40:46 2018 +0200 + + Fix sporadic failure in NestedObjectMapperTests + + Relates #34225 + +commit 81227dc3892de76b2c22781a57865bef6028a959 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Oct 4 11:31:51 2018 +1000 + + [Authz] Allow update settings action for system user (#34030) + + When the cluster.routing.allocation.disk.watermark.flood_stage watermark + is breached, DiskThresholdMonitor marks the indices as read-only. This + failed when x-pack security was present as system user does not have the privilege + for update settings action("indices:admin/settings/update"). + This commit adds the required privilege for the system user. Also added missing + debug logs when access is denied to help future debugging. + An assert statement is added to catch any missed privileges required for + system user. + + Closes #33119 + +commit 6dd716b0c40032537c8bc2910f6e35c5ae8ece04 +Author: Nhat Nguyen +Date: Wed Oct 3 21:03:24 2018 -0400 + + Replace version with reader cache key in IndicesRequestCache (#34189) + + Today we use the version of a DirectoryReader as a component of the key + of IndicesRequestCache. This usage is perfectly fine since the version + is advanced every time a new change is made into IndexWriter. In other + words, two DirectoryReaders with the same version should have the same + content. However, this invariant is only guaranteed in the context of a + single IndexWriter because the version is reset to the committed version + value when IndexWriter is re-opened. + + Since #33473, each IndexShard may have more than one IndexWriter, and + using the version of a DirectoryReader as a part of the cache key can + cause IndicesRequestCache to return stale cached values. For example, in + #27650, we rollback the engine (i.e., re-open IndexWriter), index new + documents, refresh, then make a count request, but the search layer + mistakenly returns the count of the DirectoryReader of the previous + IndexWriter because the current DirectoryReader has the same version of + the old DirectoryReader even their documents are different. This is + possible because these two readers come from different IndexWriters. + + This commit replaces the the version with the reader cache key of + IndexReader as a component of the cache key of IndicesRequestCache. + + Closes #27650 + Relates #33473 + +commit f79282e47da26b351dc6b2d97e669c0ea9287b5e +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Oct 4 08:23:59 2018 +1000 + + [TESTS] Set SO_LINGER and SO_REUSEADDR on the mock socket (#34211) + + In SessionFactoryLoadBalancingTests#testRoundRobinWithFailures() + we kill ldap servers randomly and immediately bind to that port + connecting to mock server socket. This is done to avoid someone else + listening to this port. As the creation of mock socket and binding to the + port is immediate, sometimes the earlier socket would be in TIME_WAIT state + thereby having problems with either bind or connect. + This commit sets the SO_REUSEADDR explicitly to true and also sets + the linger on time to 0(as we are not writing any data) so as to + allow re-use of the port and close immediately. + + Note: I could not find other places where this might be problematic + but looking at test runs and netstat output I do see lot of sockets + in TIME_WAIT. If we find that this needs to be addressed we can + wrap ServerSocketFactory to set these options and use that with in + memory ldap server configuration during tests. + + Closes #32190 + +commit d66a63ed7a13fa9427c6158772c3819f938b1a83 +Author: Gordon Brown +Date: Wed Oct 3 15:39:21 2018 -0600 + + [HLRC] Update Stop & Start ILM APIs to use Validatable (#34242) + + Convert the Stop and Start request objects to inherit from the + preferred Validatable instead of ActionRequest. + +commit cbe1cf98c6e72e1f4cf85d442ebcbb99e844767a +Merge: a9eae1d0687 ee21067a418 +Author: David Turner +Date: Wed Oct 3 22:12:56 2018 +0100 + + Merge branch 'master' into zen2 + +commit a21a99da1817e703c7bbf73ea76ceca66a6a6428 +Author: Jay Modi +Date: Wed Oct 3 12:31:19 2018 -0600 + + Security: upgrade unboundid ldapsdk to 4.0.8 (#34247) + + This commit upgrades the unboundid ldapsdk to version 4.0.8. The + primary driver for upgrading is a fix that prevents this library from + rewrapping Error instances that would normally bubble up to the + UncaughtExceptionHandler and terminate the JVM. Other notable changes + include some fixes related to connection handling in the library's + connection pool implementation. + + Closes #33175 + +commit f10735aa9afca7f652e0f983c12620b268a38b47 +Author: Tal Levy +Date: Wed Oct 3 11:20:43 2018 -0700 + + ILM integration test with full policy (#33402) + + - this adds an integration test that runs through a policy + with all the actions defined. + - adds a test specific to a policy having just a rollover action + - bumps the node count to 4 + +commit d45fe43a68b4706745f34533dd7d2b53614c2167 +Author: Kazuhiro Sera +Date: Thu Oct 4 02:11:39 2018 +0900 + + Fix a variety of typos and misspelled words (#32792) + +commit ee21067a418be2abab626fe847ff0eb607cc641f +Author: Jim Ferenczi +Date: Wed Oct 3 18:33:39 2018 +0200 + + Add early termination support for min/max aggregations (#33375) + + This commit adds the support to early terminate the collection of a leaf + in the min/max aggregator. If the query matches all documents the min and max value + for a numeric field can be retrieved efficiently in the points reader. + This change applies this optimization when possible. + +commit 8f10c771e62016bc156a01310ae35089b605f93d +Author: Mayya Sharipova +Date: Wed Oct 3 11:56:18 2018 -0400 + + Add migration info for missing values in script + + Relates to #30975 + +commit 3c1fdc9fc0aa757cdaa48b1217eae18dbc9a1896 +Author: Jay Modi +Date: Wed Oct 3 09:30:57 2018 -0600 + + Security: reduce memory usage of DnRoleMapper (#34250) + + The `DnRoleMapper` class is used to map distinguished names of groups + and users to role names. This mapper builds in an internal map that + maps from a `com.unboundid.ldap.sdk.DN` to a `Set`. In cases + where a lot of distinct DNs are mapped to roles, this can consume quite + a bit of memory. The majority of the memory is consumed by the DN + object. For example, a 94 character DN that has 9 relative DNs (RDN) + will retain 4KB of memory, whereas the String itself consumes less than + 250 bytes. + + In order to reduce memory usage, we can map from a normalized DN string + to a List of roles. The normalized string is actually how the DN class + determines equality with another DN and we can drop the overhead of + needing to keep all of the other objects in memory. Additionally the + use of a List provides memory savings as each HashSet is backed by a + HashMap, which consumes a great deal more memory than an appropriately + sized ArrayList. The uniqueness we get from a Set is maintained by + first building a set when parsing the file and then converting to a + list upon completion. + + Closes #34237 + +commit 90c55f5e36ddc9d7cbf2fcde8e88027db8a5e060 +Merge: 388f754a8e1 f09190c14d2 +Author: Lee Hinman +Date: Wed Oct 3 09:11:28 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit f09190c14d28c04c937e6ceb2f01b381a1369ac8 +Author: albendz <12453997+albendz@users.noreply.github.com> +Date: Wed Oct 3 07:22:01 2018 -0700 + + Require combine and reduce scripts in scripted metrics aggregation (#33452) + + * Make text message not required in constructor for slack + + * Remove unnecessary comments in test file + + * Throw exception when reduce or combine is not provided; update tests + + * Update integration tests for scripted metrics to always include reduce and combine + + * Remove some old changes from previous branches + + * Rearrange script presence checks to be earlier in build + + * Change null check order in script builder for aggregated metrics; correct test scripts in IT + + * Add breaking change details to PR + +commit 9d36cbaf162da2b00a68a9f7c1ecc9997a736a28 +Author: Jason Tedor +Date: Wed Oct 3 09:51:05 2018 -0400 + + Set BWC builds for 6.x to use JDK 11 + + The BWC builds for the 6.x branch should be using JDK 11. This commit + fixes the BWC builds to specify that they use JDK 11 instead of JDK 10 + which is now incompatible with the 6.x build. + +commit 86642d29e504fde4153008092ae0b5f2e93c4b64 +Author: Jason Tedor +Date: Wed Oct 3 08:59:00 2018 -0400 + + Require JDK 11 for compilation (#34103) + + Now that JDK 11 is GA, we would switch our 6.x and master branches to + the JDK 11 compiler. This commit makes this change, as well as removes + JDK 10 from the CI configuration. + +commit 41528c0813fe72162408051e3af29ac42b4708f7 +Author: Jim Ferenczi +Date: Wed Oct 3 14:24:01 2018 +0200 + + Adapt bwc version after backport (bis) + + Relates #34225 + +commit 1aa8e72be78e25bc2d9e5c1e85209e4e54c9f536 +Author: Jim Ferenczi +Date: Wed Oct 3 12:24:07 2018 +0200 + + Adapt bwc version after backport + + Relates #34225 + +commit 5a3e03183147e3d8f01e4110744a7185a7779a3b +Author: Jim Ferenczi +Date: Wed Oct 3 11:55:30 2018 +0200 + + Preserve the order of nested documents in the Lucene index (#34225) + + Today we reverse the initial order of the nested documents when we + index them in order to ensure that parents documents appear after + their children. This means that a query will always match nested documents + in the reverse order of their offsets in the source document. + Reversing all documents is not needed so this change ensures that parents + documents appear after their children without modifying the initial order + in each nested level. This allows to match children in the order of their + appearance in the source document which is a requirement to efficiently + implement #33587. Old indices created before this change will continue + to reverse the order of nested documents to ensure backwark compatibility. + +commit a7f62ee9026869f61af0f337746f343562769cb0 +Author: Vladimir Dolzhenko +Date: Wed Oct 3 11:37:36 2018 +0200 + + [GCE Discovery] Automatically set project-id and zone (#33721) + + Fetch default values for project-id and zone from metadata server + + Closes #13618 + +commit c6fcb60071155079d762672c01fb82ef7d044bc5 +Author: Julie Tibshirani +Date: Wed Oct 3 02:03:03 2018 -0700 + + Add support for 'ack watch' to the HLRC. (#33962) + +commit 2d64e3db9a3e86906c0db9b37f09b0a66d83b479 +Author: Colin Goodheart-Smithe +Date: Wed Oct 3 08:58:33 2018 +0100 + + Adds trace logging to IndicesRequestCache (#34180) + + * Adds trace logging to IndicesRequestCache + + This change adds trace level logging to `IndicesrrequestCache` witht eh + primary aim of helping to identify the cause of teh failures in + https://github.com/elastic/elasticsearch/issues/32827. The cache will + log at trace level when a cache hit or miss occurs including the reader + version and the cache key. Note that this change adds a + `cacheKeyRenderer` whcih supplies a human readable String of the cache + key since the actual cache key itself is a `BytesReference` containing + the wire protocol serialised form of the request. + + Logging is also added for the case where a search timeout occurs and fr + that reason the cache entry is invalidated. + + * Adds comment to remaind us to remove cacheKeyRenderer + +commit a9eae1d0687fd93170ac3f79cd5f06b0c67961b5 +Merge: a127805b4a6 d7893fd1e45 +Author: David Turner +Date: Wed Oct 3 08:36:34 2018 +0100 + + Merge branch 'master' into zen2 + +commit 388f754a8e190237c1e4842b84be919c875baaf3 +Author: Lee Hinman +Date: Tue Oct 2 20:02:50 2018 -0600 + + Change step execution flow to be deliberate about type (#34126) + + This commit changes the way that step execution flows. Rather than have any step + run when the cluster state changes or the periodic scheduler fires, this now + runs the different types of steps at different times. + + `AsyncWaitStep` is run at a periodic manner, ie, every 10 minutes by default + `ClusterStateActionStep` and `ClusterStateWaitStep` are run every time the + cluster state changes. + `AsyncActionStep` is now run only after the cluster state has been transitioned + into a new step. This prevents these non-idempotent steps from running at the + same time. It addition to being run when transitioned into, this is also run + when a node is newly elected master (only if set as the current step) so that + master failover does not fail to run the step. + + This also changes the `RolloverStep` from an `AsyncActionStep` to an + `AsyncWaitStep` so that it can run periodically. + + Relates to #29823 + +commit d7893fd1e455ecf2c2a067792283a3304bebfe5b +Author: Nhat Nguyen +Date: Tue Oct 2 17:19:53 2018 -0400 + + TEST: Mute testFollowIndexAndCloseNode + + Tracked at #33337 + +commit 6fb9c63ed6b787114a5d79342f09be748f578f28 +Author: Christoph Büscher +Date: Tue Oct 2 23:13:08 2018 +0200 + + [Docs] Fix broken link for HLRC rethrottle + +commit 5af6ae564d080c7c7c77626d88b7b2ec6381eeea +Author: Wilder Pereira +Date: Tue Oct 2 18:04:19 2018 -0300 + + Change "REST Verb" to "HTTP Verb" (#34195) + +commit a1c441f78a29fe3b86b8f7947e8a58131769b435 +Author: Christoph Büscher +Date: Tue Oct 2 21:44:15 2018 +0200 + + HLRC: Add throttling for update & delete-by-query (#33951) + + This change adds throttling to the update-by-query and delete-by-query cases + similar to throttling for reindex. This mostly means additional methods on the + client class itself, since the request hits the same RestHandler, just with + slightly different endpoints, and also the return values are similar. + +commit fb907706ecea4c8894075295f0aa148efe01adbf +Merge: 3afb8e8e661 dd3fe92673f +Author: Gordon Brown +Date: Tue Oct 2 13:43:46 2018 -0600 + + Merge branch 'master' into index-lifecycle + +commit dd3fe92673f954504dadc2b53a6168c38363ec15 +Author: Gordon Brown +Date: Tue Oct 2 13:36:13 2018 -0600 + + [DOCS] Note that User Cluster Metadata is not private (#34156) + + As user-defined cluster metadata is accessible to anyone with access to + get the cluster settings, stored in the logs, and likely to be tracked + by monitoring solutions, it is useful to clarify in the documentation + that it should not be used to store secret information. + +commit f12e0a8398880ae221c091deccf93053e73ae625 +Author: Dimitrios Liappis +Date: Tue Oct 2 21:15:58 2018 +0300 + + Add ES version 6.4.3 (#34239) + + Version bump + +commit a7ce4b31edf172ded3de691ec1c3f3fb7f73067b +Author: David Turner +Date: Tue Oct 2 19:08:19 2018 +0100 + + Fix logging of cluster state update descriptions (#34182) + + In #28941 we changed the computation of cluster state task descriptions but + this introduced a bug in which we only log the empty descriptions (rather than + the non-empty ones). This change fixes that. + +commit d3a4fe9a8e86d8040d791409111ce2e8b373fd84 +Author: Nik Everett +Date: Tue Oct 2 12:25:58 2018 -0400 + + Docs: Wrap expert script example to fit in docs (#34201) + + This slightly reworks the expert script plugin example so it fits on the + page when the docs are rendered. The box in which it is rendered is not + very wide so it took a bit of twisting to make it readable. + +commit 5140f992b4de2e3ee3df0ffd12f02ad964bfe71e +Author: Jason Tedor +Date: Tue Oct 2 12:10:43 2018 -0400 + + Fix use of hostname in Windows service (#34193) + + To pass the HOSTNAME envrionment variable to the Windows service, we + have to add some command line flags to the service invocation. Namely, + we have to specify that we are passing HOSTNAME variable, and we will + pass for it the value of %%COMPUTERNAME%%. This ensures that if the + hostname is changed, we pick this up the next time that the service is + started. This change is needed for the service now that we use the + HOSTNAME as the default node name. + +commit 2e5945a5e9fb49bcc89872980eba26078bc309a6 +Author: Jay Modi +Date: Tue Oct 2 10:10:32 2018 -0600 + + HLRC: PutUserRequest should not be closeable (#34196) + + The PutUserRequest implemented closeable as it assumed ownership of the + password provided to the class. This change removes the ownership of + the password, documents it in the javadoc, and removes the closeable + implementation. + + Additionally, the intermediate bytes used for writing the password to + XContent are now cleared. This makes the PutUserRequest consistent with + the behavior discussed in #33509. + +commit 306e178d83fef5f77c9797fae8357c294fb30f5e +Author: jaymode +Date: Tue Oct 2 09:58:33 2018 -0600 + + Test: remove awaitsfix incorrectly added in #34148 + +commit 7f5c2f10504efa640f8ac329306f010c02f431e6 +Author: Martijn van Groningen +Date: Tue Oct 2 18:01:06 2018 +0200 + + [CCR] Validate follower index historyUUIDs (#34078) + + The follower index shard history UUID will be fetched from the indices stats api when the shard follow task starts and will be provided with the bulk shard operation requests. The bulk shard operations api will fail if the provided history uuid is unequal to the actual history uuid. + + No longer record the leader history uuid in shard follow task params, but rather use the leader history UUIDs directly from follower index's custom metadata. The resume follow api will remain to fail if leader index shard history UUIDs are missing. + + Closes #33956 + +commit 8539fb68d9e15db69d6567b0f524c3b19398fe2f +Author: Jay Modi +Date: Tue Oct 2 09:54:21 2018 -0600 + + Test: Revert pinning MockWebServer to TLSv1.2 (#34148) + + Revert "[TESTS] Pin MockWebServer to TLS1.2 (#33127)" (commit + 214652d4af8188d4ba872626eeea3bcdff7096f0) and "Pin TLS1.2 in + SSLConfigurationReloaderTests" (commit + d9f5e4fd2e06c9b69f3b4744e49e747e1ff708b4), which pinned the + MockWebServer used in the SSLConfigurationReloaderTests to TLSv1.2 in + order to prevent failures with JDK 11 related to ssl session + invalidation. We no longer need this pinning as the problematic code + was fixed in #34130. + +commit 6b714c9e1e0747e1f8a389b2c6d3abf016091894 +Author: Andriy +Date: Tue Oct 2 18:45:30 2018 +0300 + + [Docs] Updated link to kafka-elasticsearch-consumer project (#34234) + +commit a4cf4ca585920c2001ce33beff2b4a84e3a2d243 +Author: Lisa Cawley +Date: Tue Oct 2 08:37:45 2018 -0700 + + [DOCS] Clarifies examples in reindex and task APIs (#33143) + +commit 13af5d5d7f45f27746aacf24aa73ef2faafd83aa +Author: Serge Populov +Date: Tue Oct 2 19:29:32 2018 +0500 + + Docs: Fix typo in field name in aggregations (#34223) + +commit 3afb8e8e661f6d5c4e9e81d780e9ca2f56bbfeed +Author: Alpar Torok +Date: Tue Oct 2 17:11:01 2018 +0300 + + Add Gradle attributes to CI jobs (#34178) + + Maintain the additional arguments in the branch so CI doesn't have to care. + +commit 5183ea3d68a325c25a9681483d2756c6ee692e88 +Author: Christoph Büscher +Date: Tue Oct 2 15:58:07 2018 +0200 + + Use OptionalInt instead of Optional (#34220) + + Optionals containing boxed primitive types are prohibitively costly because they + have two level of boxing. For Optional the analogous OptionalInt can be + used to avoid the boxing of the contained int value. + +commit 2ba18f50a87be31d9bcb28780f1ee3fc903579e9 +Author: Marios Trivyzas +Date: Tue Oct 2 15:26:04 2018 +0200 + + SQL: Remove more ANTLR4 grammar ambiguities (#34074) + + The `-` and `+` as a number literal prefix are already + parsed by the rule in `valueExpression`. To accommodate + this, there are some code changes that enables the + `ExpressionBuilder` to parse Literal integers and decimals + together with the `-/+` prefix sign (if exists) and validate + them (wrong format, large numbers, etc.). + + Follows: #33854 + +commit 10201e06cbf0d4e8ecf6db497a9a0e9ba110a048 +Author: Benjamin Trent +Date: Tue Oct 2 06:21:46 2018 -0700 + + Allowing {index}/_xpack/rollup/data to accept comma delimited list (#34115) + + * Allowing `{index}/_xpack/rollup/data` to accept comma delimited list + + * Address PR comments + +commit f904c41506e0161d682bc03adddb04b0bfb89369 +Author: Nik Everett +Date: Tue Oct 2 09:11:29 2018 -0400 + + HLRC: Add get rollup job (#33921) + + Adds support for the get rollup job to the High Level REST Client. I had + to do three interesting and unexpected things: + 1. I ported the rollup state wiping code into the high level client + tests. I'll move this into the test framework in a followup and remove + the x-pack version. + 2. The `timeout` in the rollup config was serialized using the + `toString` representation of `TimeValue` which produces fractional time + values which are more human readable but aren't supported by parsing. So + I switched it to `getStringRep`. + 3. Refactor the xcontent round trip testing utilities so we can test + parsing of classes that don't implements `ToXContent`. + +commit 7dbc403226afbd8718bb2750a5b8b9ee3c66ae31 +Author: Nhat Nguyen +Date: Tue Oct 2 09:04:03 2018 -0400 + + TEST: Index diff num docs in rolling upgrade tests (#34191) + + Today we index the same number of documents (50 documents) in each round + of the rolling upgrade tests. If the actual count does not match, we can + not guess the problematic round. + + Relates #27650 + +commit ead6ffce54ceed45b347dd6d8a1021d46b634cee +Author: Jim Ferenczi +Date: Tue Oct 2 14:53:26 2018 +0200 + + Fix cross fields mode of the query_string query (#34216) + + This change fixes a bug in the cross fields mode of the `query_string` + query. The multi fields query builder must be reseted before parsing + in order to clear the list of expanded fields coming from the previous text block. + + Closes #34215 + +commit 3f8cc89c9f667cd3012bf8e813285f2e5d3a4485 +Author: Przemyslaw Gomulka +Date: Tue Oct 2 14:32:56 2018 +0200 + + Completion types with multi-fields support (#34081) + + Mappings with completion type and multi-fields, were not able to index array or + object format on completion fields. Only string format was supported. + This is fixed by providing multiField parser with externalValueContext with already parsed object + + closes #15115 + +commit b1b0f3276b060c287e465fcbfdf47cbc6d40e362 +Author: Alexander Reelsen +Date: Tue Oct 2 14:13:30 2018 +0200 + + Core: Add methods to get locale/timezone in DateFormatter (#34113) + + This adds some method into the `DateFormatter` interface, namely + + * `withLocale()` to change the locale of a date formatter + * `getLocale()` + * `getZone()` + * `hashCode()` + * `equals()` + + These methods will be needed for aggregations and mapping changes, where + zones and locales can be specified in the mapping or in search/aggs + parts of a search request. + +commit 1d049cadbe4f29b1cad8643e0505c6e514d4191d +Author: Ioannis Kakavas +Date: Tue Oct 2 13:23:13 2018 +0300 + + Fix HLRC docs + +commit a127805b4a6c44741db36c1abf567945dc61e640 +Author: David Turner +Date: Tue Oct 2 11:22:05 2018 +0100 + + [Zen2] Simulate scheduling delays (#34181) + + Today we schedule tasks (both immediate and future ones) exactly when + requested. In fact it is more realistic to allow for a small amount of delay in + the scheduling of tasks, and this helps to exercise more interleavings of + actions and therefore to improve test coverage. + + This change adds to the DeterministicTaskQueue the ability to add a random + delay to the scheduling of tasks. + + This change also provides more explicit timeouts for stabilisation in the + CoordinatorTests. + + Using the randomised scheduling feature in the CoordinatorTests also found a + situation in which we could become a leader, then a candidate, and then a + leader again very quickly, causing a clash of the _BECOME_MASTER_ and + _FINISH_ELECTION_ tasks. We change their behaviour to not consider these + duplicates to be problematic. + +commit 3eed873dde42c47b4b3b29c86b741fc250b12d65 +Author: Shaunak Kashyap +Date: Tue Oct 2 03:19:12 2018 -0700 + + Updating test assertion (#34040) + +commit 300896d401eae0d0fa9333fa0967fbe7f7be53d3 +Author: Ioannis Kakavas +Date: Tue Oct 2 12:14:25 2018 +0300 + + HLRC: add change password API support (#33509) + + This change adds support for the change password APIs to the high + level rest client. + + Relates #29827 + +commit a8a2a83d457c6d317c1c52ea0965f497ae565aab +Author: Marios Trivyzas +Date: Tue Oct 2 11:47:42 2018 +0300 + + SQL: Fix grammar for `*` in arithm expressions (#34176) + + Previously, parsing an arithmetic expression with `*` and no spaces, + e.g.: `2*i` threw a parsing exception as the grammar rule for + tableIdentifier was clashing with the rule for arithmetic operator `*`. + + This issue comes already in the lexer and the left part of the + expression (in our example `2*`) was recognised as a + TABLE_IDENTIFIER token. + + The solution adopted is to allow the `*` wildcard in the table name + only if it's surrounded with double quotes, e.g.: `"my*index"` + + Closes: #33957 + +commit aba4a59d0d45d1455dff1a2c8c5d83a617032c01 +Author: Jim Ferenczi +Date: Tue Oct 2 09:30:23 2018 +0200 + + Handle terms query when detecting if a query can match nested docs (#34072) + + When nested objects are present in the mappings, we add a filter in + queries to exclude them if there is no evidence that the query cannot + match in this space. In 6x we visit the query in order to find a mandatory + clause that can match root documents only. If we find one we can omit the + nested documents filter. Currently only `term` and `range` queries are checked, + this change adds the support for `terms` query to effectively remove the nested filter + if a mandatory `terms` clause targets a non-nested field. + + Closes #34067 + +commit 2aff005a69601d4431429fc5ea4e62529be73695 +Author: David Turner +Date: Tue Oct 2 03:17:55 2018 +0100 + + Clean up TransportMasterNodeAction (#34076) + + Mainly this fixes a warning by replacing the unchecked `new ActionListener` + with the checked `new ActionListener`, and it also fixes the line + length violations in this class. + +commit eb1113ba78c7e4d33759e4911abe28ee1f199a1c +Author: Marios Trivyzas +Date: Tue Oct 2 02:06:51 2018 +0300 + + SQL: Fix function resolution (#34137) + + Remove CamelCase to CAMEL_CASE conversion when resolving + a function. Only convert user input to upper case and then + try to match with aliases or primary names. + + Keep the internal conversion FunctionName to FUNCTION__NAME + which provides flexibility when registering functions by their class + name. + + Fixes: #34114 + +commit 2d9cb214901658515e5b1f012df8969a7bc0bd9e +Merge: a49d59802a3 ad3218b4abd +Author: Lee Hinman +Date: Mon Oct 1 14:10:09 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit a49d59802a3c277e9315e21c1e5a24c734d9e778 +Author: Lee Hinman +Date: Mon Oct 1 13:45:26 2018 -0600 + + Use more descriptive task names for ILM cluster state updates (#34161) + + Rather than using "ILM" for everything, we should use more descriptive names so + debugging from logs is easier to do. + + Resolves #34118 + +commit ad3218b4abd3aafdad0f7fb50ac05cfc74d8b34b +Author: Nik Everett +Date: Mon Oct 1 15:22:15 2018 -0400 + + Checkstyle: Catch wide snippets (#34163) + + We use wrap code in `// tag` and `//end` to include it in our docs. Our + current docs style wraps code snippets in a box that is only wide enough + for 76 characters and adds a horizontal scroll bar for wider snippets + which makes the snippet much harder to read. This adds a checkstyle check + that looks for java code that is included in the docs and is wider than + that 76 characters so all snippets fit into the box. It solves many of + the failures that this catches but suppresses many more. I will clean + those up in a follow up change. + +commit 2923fb5b31124f8c23d7aea857c030baf87511b2 +Author: Christophe Bismuth +Date: Mon Oct 1 20:49:08 2018 +0200 + + Disallow "enabled" attribute change for types in mapping update (#33933) + + This commit adds a check for "enabled" attribute change for types when + a RestPutMappingAction is received. A MappingException is thrown when + such a change is detected. Change are prevented in both ways: "false -> true" + and "true -> false". + + Closes #33566 + +commit d12a64eac2064348dabc5b7a424e96e81007e0d8 +Author: Martijn van Groningen +Date: Mon Oct 1 20:13:16 2018 +0200 + + [CCR] Only use primary shards and get expected count from leader index (#34186) + + Closes #34173 + +commit a02debadfe8ee1afbd5e5ea3d70af1f8362bf5ab +Author: Nhat Nguyen +Date: Mon Oct 1 11:52:59 2018 -0400 + + TEST: Unmute testFollowIndexAndCloseNode + + Since #34099, the FollowingEngine will skip an operation which was + already processed before. With that change, it should be okay to unmute + testFollowIndexAndCloseNode. + +commit 84111e9607f01e0f514b07bcdd03afd0143b0361 +Author: Vladimir Dolzhenko +Date: Mon Oct 1 17:54:32 2018 +0200 + + fix broken doc due to `elasticsearch-translog` removal + +commit 96be057195afa49be4382b376a5e696d09fa6cd9 +Author: Benjamin Trent +Date: Mon Oct 1 08:09:46 2018 -0700 + + Removing unused ML parameters (#34159) + +commit a1d2ded98d9c5d98d6e75a70f5ea66c580d5aeae +Author: David Roberts +Date: Mon Oct 1 15:35:37 2018 +0100 + + [ML] Fix unit test deadlock problem (#34174) + + This change fixes a potential deadlock problem in the unit + test introduced in #34117. + + It also removes a piece of debug code and corrects a docs + formatting problem that were both added in that same PR. + +commit 2e2ae19b97f7e94a68f57b07c5e53bdb2dd5a684 +Author: Vladimir Dolzhenko +Date: Mon Oct 1 16:21:14 2018 +0200 + + drop elasticsearch-translog for 7.0 (#33373) + + + #32281 adds elasticsearch-shard to provide bwc version of elasticsearch-translog for 6.x; have to remove elasticsearch-translog for 7.0 + + Relates to #31389 + +commit 8082b4ad4a9fbefd74eb57c987ac26374da449e1 +Author: Nik Everett +Date: Mon Oct 1 09:59:36 2018 -0400 + + Docs: DRY up HLRC index docs (#34157) + + Use tricks pioneered in #34125 to shorted asciidoc for the index API's + docs. Also slightly shorten the supported-api docs file. + +commit d25e3ef065f63fac4d4ff8d26252cb504ac12385 +Author: Alan Woodward +Date: Mon Oct 1 09:45:42 2018 +0100 + + [CI] Use a single shard for synonym REST tests, to ensure score ordering + is stable + +commit 17e6932bf3ceb97fbea7be1fee8d4b1218df1205 +Author: Christoph Büscher +Date: Mon Oct 1 10:29:19 2018 +0200 + + [Tests] Rename DocumentMapperMergeTests (#34121) + + Renaming to simply DocumentMapperTests to indicate this is where other unit + tests should go. Also removing outdates Todo in DocumentMapperParserTests. + +commit 80f7c1dcc941fc9888a2ba00b44a0e65ef974b14 +Author: Jason Tedor +Date: Sun Sep 30 14:25:50 2018 -0400 + + Fix compilation in unfollow action tests + + This arose when two commits were pushed at roughly the same time, both + of which compiled successfully against master, but not when taken + together. This commit fixes a reference in one of the commits that was + changed in the other commit. + +commit 189376505516cae4350403d62008008e54b6104b +Author: Jason Tedor +Date: Sun Sep 30 14:29:32 2018 -0400 + + Change CCR stats endpoint to be index-centric (#34169) + + This commit modifies the CCR stats endpoint for indices to be + /{index}/_ccr/stats. This makes this endpoint consistent with other + index-centric endpoints like indices stats. + +commit e2bd2028d8dbf68407b773193343eee3af39b875 +Author: Jason Tedor +Date: Sun Sep 30 14:22:22 2018 -0400 + + Allow specifying shard changes batch sizes in bytes (#34168) + + This commit changes the shard changes requests from using a raw byte + value to being able to be specified using bytes units (e.g., 4mb). + +commit 7c91c7a6381e4db757fae2192530a622244847b0 +Author: Martijn van Groningen +Date: Sun Sep 30 19:31:30 2018 +0200 + + fixed test compile error + +commit b1a27b2e6b16a5bba19f331b48378857e4f76984 +Author: Martijn van Groningen +Date: Sun Sep 30 19:19:34 2018 +0200 + + [CCR] Add unfollow API (#34132) + + The unfollow API changes a follower index into a regular index, so that it will accept write requests from clients. + + For the unfollow api to work the index follow needs to be stopped and the index needs to be closed. + + Closes #33931 + +commit fb85dd8c0c0fcc2570f557557c5885b386fe6eb6 +Author: Hendrik Muhs +Date: Sat Sep 29 15:18:00 2018 +0200 + + [Docs] remove double words (#34138) + + removes double wording and typo + +commit ad613988797746d7846a606372d00caaf1e475ef +Author: Nhat Nguyen +Date: Fri Sep 28 20:42:26 2018 -0400 + + CCR: Optimize indexing ops using seq_no on followers (#34099) + + This change introduces the indexing optimization using sequence numbers + in the FollowingEngine. This optimization uses the max_seq_no_updates + which is tracked on the primary of the leader and replicated to replicas + and followers. + + Relates #33656 + +commit 47cbae9b265c3ad86a9e8de28d8b416539354e6b +Author: Ryan Ernst +Date: Fri Sep 28 17:13:08 2018 -0700 + + Scripting: Remove ExecutableScript (#34154) + + This commit removes the legacy ExecutableScript, which was no longer + used except in tests. All uses have previously been converted to script + contexts. + +commit 6ea396a476ada0c6d5ff0f27a3e5526b42ff1502 +Merge: a26cc1a2422 128ea1b7b51 +Author: Lee Hinman +Date: Fri Sep 28 15:40:12 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 128ea1b7b5139d7fb22275c1105dd3da70722954 +Author: Jason Tedor +Date: Fri Sep 28 17:33:29 2018 -0400 + + Add Ubuntu 18.04 to packaging tests (#34139) + + We intend to start supporting Ubuntu 18.04 with the 6.5.0 release. This + commit adds Ubuntu 18.04 to the packaging tests. + +commit 412face402a4951bba6834f9e8a33e89144f6ca7 +Author: Yannick Welsch +Date: Fri Sep 28 23:12:59 2018 +0200 + + Move NodeRemovalClusterStateTaskExecutor out of ZenDiscovery (#34147) + + Allows this class to be cleanly shared between Zen1 and Zen2. Follow-up to #33917 + +commit 47bcb56a89d6a00a5596dfeba46428a7d51c1c41 +Author: Nik Everett +Date: Fri Sep 28 14:48:11 2018 -0400 + + Docs: Cut down on high level rest client copy-and-paste-ness (#34125) + + * Anchors + + * includes + + * get settings + + * more + + * More + +commit 68b42f7d893ffc66340f0da95ffb4a9882a6f08f +Author: Lisa Cawley +Date: Fri Sep 28 10:17:41 2018 -0700 + + [DOCS] Moves reserved keywords under SQL section (#34124) + +commit 76dd3948f3c1c1016d156fefb0dbe88facdd40b4 +Author: Armin Braun +Date: Fri Sep 28 19:09:49 2018 +0200 + + TESTS: Relax Assertion About Deleting Shard Dir (#34120) + + * TESTS: Relax Assertion About Deleting Shard Dir + + * Allow empty state directory to prevent test from failing + * Closes #32686 + +commit 57052f617aa00fb2e002b3d16bec91b20bb73d98 +Author: lcawl +Date: Fri Sep 28 10:06:02 2018 -0700 + + [DOCS] Fixes callout in ML API + +commit f143e39a71bcf34b07ff1b2a6af95093b80ba58b +Author: Lisa Cawley +Date: Fri Sep 28 09:41:14 2018 -0700 + + [DOCS] Adds new dynamic machine learning settings (#34094) + +commit f709c2f6946e92b9f864d001286247d10864230e +Author: David Roberts +Date: Fri Sep 28 17:32:35 2018 +0100 + + [ML] Add a timeout option to file structure finder (#34117) + + This can be used to restrict the amount of CPU a single + structure finder request can use. + + The timeout is not implemented precisely, so requests + may run for slightly longer than the timeout before + aborting. + + The default is 25 seconds, which is a little below + Kibana's default timeout of 30 seconds for calls to + Elasticsearch APIs. + +commit bc84c8ecd6bbd91b7187d4b9d237982a1fe34b32 +Author: Jack Conradson +Date: Fri Sep 28 09:11:41 2018 -0700 + + Painless: Remove caching of Painless scripts in GenericElasticsearchScript (#34116) + + When a script uses the special-cased SearchScript or ExecutableScript, Painless is + internally caching the GenericElasticsearchScript that is generated via compile as part of + the newInstance method for the factories. This breaks the model for class bindings as the + class binding expects to not have any stored information for each instance generated via + newInstance from the factory. This change fixes the issue by creating a new instance of + the GenericElasticsearchScript each time newInstance is called against the factory. + +commit a984f8afb385a75e467a3edbebcd3956d21436c4 +Author: Martijn van Groningen +Date: Fri Sep 28 17:51:23 2018 +0200 + + [CCR] Validate index privileges prior to following an index (#33758) + + Prior to following an index in the follow API, check whether current + user has sufficient privileges in the leader cluster to read and + monitor the leader index. + + Also check this in the create and follow API prior to creating the + follow index. + + Also introduced READ_CCR cluster privilege that include the minimal + cluster level actions that are required for ccr in the leader cluster. + So a user can follow indices in a cluster, but not use the ccr admin APIs. + + Closes #33553 + + Co-authored-by: Jason Tedor + +commit 3d7e3b2ab13f4f18c820cc7927bff4340cdcd6f6 +Author: Martijn van Groningen +Date: Fri Sep 28 17:43:42 2018 +0200 + + [TEST] changed naming of test methods to not refer to old api names. + +commit 7e04a2bdf7713dcf225ec5d99ec12dadde7ed2fb +Author: Jay Modi +Date: Fri Sep 28 09:13:11 2018 -0600 + + Update SSLTrustRestrictionTests for JDK11 (#34131) + + In prior versions of Java, we expected to see a SSLHandshakeException + when starting a handshake with a server that we do not trust. In JDK11, + the exception has changed to a SSLException, which + SSLHandshakeException extends. This is most likely a side effect of the + TLS 1.3 changes in JDK11. This change updates the test to catch the + SSLException instead of the SSLHandshakeException and enables the test + to work on JDK8 through JDK11. + + Closes #29989 + +commit 14d841ef219a941a659d330e444c71e811c26ff6 +Author: Jay Modi +Date: Fri Sep 28 09:03:35 2018 -0600 + + Handle null SSLSessions during invalidation (#34130) + + The SSLService invalidates SSLSessions when there is a change to any of + the underlying key or trust material. However, this invalidation code + did not check for a null SSLSession being returned from the context and + assumed that the context would always return a non-null object. The + return of a null object is possible in all versions, but JDK11 seems to + return them more often due to changes for TLS 1.3. There are a number + of reasons that we get a id of a session but the context returns null + when the session with that id is requested. Some of the reasons for + this are: + + * Session was evicted by session cache + * Session has timed out + * Session has been invalidated by another caller + + To handle this, the SSLService now checks if the value is null before + calling invalidate on the SSLSession. + + Closes #32124 + +commit 95977f4db9f5ccf21278adcc4e117e508ede6b29 +Author: Ryan Ernst +Date: Fri Sep 28 07:58:17 2018 -0700 + + Scripting: Add watcher script contexts (#34059) + + This commit removes the use of ExecutableScript from watcher in favor of + custom script contexts for both watcher condition scripts and transform + scripts. + +commit 0c3846d3d57e4fb6cdee7824c9078a280e859d9b +Author: Jay Modi +Date: Fri Sep 28 08:46:14 2018 -0600 + + Update SSLReloadIntegTests to work with JDK11 (#34129) + + In prior versions of Java, we expected to see a SSLHandshakeException + when starting a handshake with a server that we do not trust. In JDK11, + the exception has changed to a SSLException, which + SSLHandshakeException extends. This is most likely a side effect of the + TLS 1.3 changes in JDK11. This change updates the test to catch the + SSLException instead of the SSLHandshakeException and enables the test + to work on JDK8 through JDK11. + + Closes #32293 + +commit 99681f91f8ac41967637e1e75fb381e451368f8d +Author: Jason Tedor +Date: Fri Sep 28 10:05:04 2018 -0400 + + Use more precise does S3 bucket exist method (#34123) + + We are using a deprecated method for checking if an S3 bucket + exists. This deprecated method has a limitation that it can not + distinguish between invalid credentials and a lack of permissions. This + commit switches to using a method that correctly surfaces if invalid + credentials are supplied when checking for the existence of a bucket. + +commit b3218fef200cef5411101b2f0ce0785c87e69dcf +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Fri Sep 28 15:25:19 2018 +0200 + + LLREST: Introduce a strict mode (#33708) + + Introduces `RestClientBuilder#setStrictDeprecationMode` which defaults + to false but when set to true, causes a rest request to fail if a + deprecation warning header comes back in the response from Elasticsearch. + This should be valueable to Elasticsearch's tests, especially those of the + High Level REST Client where they will help catch divergence between the + client and the server. + +commit eb00348b57ccc2def0d3865e254828f75664794d +Author: Martijn van Groningen +Date: Fri Sep 28 13:33:09 2018 +0200 + + [CCR] Adjust list retryable errors (#33985) + + The following changes were made: + * Added ElasticsearchSecurityException. For in the case the current user has insufficient privileges while an index is being followed. Prior to following ccr checks whether the current user has sufficient privileges and if not the follow api fails with an error. + * Added Index block exception. If the leader index gets closed, this exception is returned. + * Added ClusterBlockException service unavailable. In case for example the leader cluster is without elected master. + * Removed IndexNotFoundException. If the leader / follower index has been deleted, ccr will need to stop the shard follow tasks with an error. + + Closes #33954 + +commit e2f310b56cf4f5c45196f7adda1cd687d164ef3d +Author: Hendrik Muhs +Date: Fri Sep 28 13:30:50 2018 +0200 + + Fix AggregationFactories.Builder equality and hash regarding order (#34005) + + Fixes the equals and hash function to ignore the order of aggregations to ensure equality after serialization + and deserialization. This ensures storing configs with aggregation works properly. + + This also addresses a potential issue in caching when the same query contains aggregations but in + different order. 1st it will not hit in the cache, 2nd cache objects which shall be equal might end up twice in + the cache. + +commit 980cfc69d64bc339d8ce58ff7ab7a24bce5cd1fa +Author: David Turner +Date: Fri Sep 28 12:29:34 2018 +0100 + + Integrate FollowerChecker with Coordinator (#34075) + + This change ensures that the leader node periodically checks that its followers + are healthy, and that they are removed from the cluster if not. + +commit c4b831645cb0369449d092a14ecaff4c04528141 +Author: Armin Braun +Date: Fri Sep 28 12:40:20 2018 +0200 + + MINOR: Remove some deadcode in NodeEnv and Related (#34133) + +commit 94df8ec3585582823c6c18a29252bb59022693f5 +Author: Russ Cam +Date: Fri Sep 28 19:20:51 2018 +1000 + + Rest-Api-Spec: Correct spelling in filter_path description (#33154) + + This commit corrects the spelling of response in the description of + filter_path in the REST API spec. + +commit bc7d69f74aeaee11dfdb33a1e6cf37a1fcabbe93 +Author: Alexander Reelsen +Date: Fri Sep 28 10:53:33 2018 +0200 + + Core: Don't rely on java time for epoch seconds formatting (#34086) + + In order to be compatible with joda time, this adds an epoch seconds + formatter, that is able to parse floating point values. + + However joda time discards the floating point values, but still parses + the data, where as this one is able to parse the whole value including + milliseconds. + +commit 506c1c2d47dff379dfcb7ef0962e6373f0f1b451 +Author: Martijn van Groningen +Date: Fri Sep 28 10:34:08 2018 +0200 + + Retry errors when fetching follower global checkpoint. (#34019) + + Closes #34016 + +commit d846f587725a2af9e8ee39ab3ae66e89f5af3f1e +Author: Alexander Reelsen +Date: Fri Sep 28 10:22:43 2018 +0200 + + Watcher: Reenable watcher stats REST tests (#34107) + + Due to a bug, that was fixed in #33360 and commit + 1de2a925ce8bf30d53ad71dfcd6e33ebbf7827c6 the initial adding of a watch + could get lost, thus leaving the watcher stats count as zero despite adding a watch. + + Closes #33326 + +commit f243d75f5920620697906c4d7e923c8dada09141 +Author: Alan Woodward +Date: Fri Sep 28 09:02:47 2018 +0100 + + Remove special-casing of Synonym filters in AnalysisRegistry (#34034) + + The synonym filters no longer need access to the AnalysisRegistry in their + constructors, so we can remove the special-case code and move them to the + common analysis module. + + This commit means that synonyms are no longer available for `server` integration tests, + so several of these are either rewritten or migrated to the common analysis module + as rest-spec-api tests + +commit 9129948f60eec362d4e62deab675c1bbe034b5c4 +Author: Martijn van Groningen +Date: Fri Sep 28 08:02:20 2018 +0200 + + Rename CCR APIs (#34027) + + * Renamed CCR APIs + + Renamed: + * `/{index}/_ccr/create_and_follow` to `/{index}/_ccr/follow` + * `/{index}/_ccr/unfollow` to `/{index}/_ccr/pause_follow` + * `/{index}/_ccr/follow` to `/{index}/_ccr/resume_follow` + + Relates to #33931 + +commit 17b3b97899f9d0fbd720ce2e2c50ac40235557b3 +Author: Martijn van Groningen +Date: Fri Sep 28 07:45:32 2018 +0200 + + Fixed CCR stats api serialization issues and (#33983) + + always use `IndicesOptions.strictExpand()` for indices options. + + The follow index may be closed and we still want to get stats from + shard follow task and the whether the provided index name matches with + follow index name is checked when locating the task itself in the ccr + stats transport action. + +commit 9cd4f70a67428752c94a70c3d2396a3cf2e3c586 +Author: Julie Tibshirani +Date: Thu Sep 27 20:59:03 2018 -0700 + + Support 'string'-style queries on metadata fields when reasonable. (#34089) + + * Make sure 'ignored' and 'routing' field types inherit from StringFieldType. + * Add tests for prefix and regexp queries. + * Support prefix and regexp queries on _index fields. + +commit 33a264a408ea6f7fc41ae17350a0ca7626b5a814 +Author: Nik Everett +Date: Thu Sep 27 17:35:04 2018 -0400 + + Logging: Drop Settings from security logger get calls (#33940) + + `Settings` is no longer required to get a `Logger` and we went to quite + a bit of effort to pass it to the `Logger` getters. This removes the + `Settings` from all of the logger fetches in security and x-pack:core. + +commit 15515a616ebb8dc9d1fa100baab2641b09b92776 (es/feature/rollups) +Author: Costin Leau +Date: Thu Sep 27 23:42:57 2018 +0300 + + SQL: Internal refactoring of operators as functions (#34097) + + Centralize and simplify the script generation between operators and functions which are currently decoupled. As part of this process most predicates (<, <=, etc...) were made ScalarFunction as their purpose and functionality is quite similar (see % and MOD functions). + Renamed ProcessDefinition to Pipe + Add ScriptWeaver as a mixin-in interface for script customization + Add logic for equals/lte/lt + Improve BinaryOperator/expression toString + Minimize duplication across string functions + + Close #33975 + +commit a2c941806b018dcdb01acdcfba291a2e988d9839 +Author: Ryan Ernst +Date: Thu Sep 27 12:23:59 2018 -0700 + + Tests: Add support for custom contexts to mock scripts (#34100) + + This commit adds the ability to plug in compilation of custom contexts + in mock script engine. This is needed for testing plugins which add + custom contexts like watcher. + +commit 73ee721b29b424f002100a40c6e4d1eb5ebf84fd +Author: Jake Landis +Date: Thu Sep 27 13:54:26 2018 -0500 + + ingest: correctly measure chained pipeline stats (#33912) + + Prior to this change when a pipeline processor called another + pipeline, only the stats for the first processor were recorded. + The stats for the subsequent pipelines were ignored. This change + properly accounts for pipelines irregardless if they are the first + or subsequently called pipelines. + + This change moves the state of the stats from the IngestService + to the pipeline itself. Cluster updates are safe since the pipelines + map is atomically swapped, and if a cluster update happens + while iterating over stats (now read directly from the pipeline) + a slightly stale view of stats may be shown. + +commit a26cc1a2422c7d6eda7d0c204f7588d79fa597e6 +Merge: c0bfc07f533 37be3e713cb +Author: Lee Hinman +Date: Thu Sep 27 11:00:37 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 37be3e713cb8bae4c6dffa5405da19ea0569fbe9 +Author: Lisa Cawley +Date: Thu Sep 27 08:41:38 2018 -0700 + + [DOCS] Synchronize location of Breaking Changes (#33588) + +commit 949e4e9d1a4b37ded617c78a493039009b9e2879 +Author: Lisa Cawley +Date: Thu Sep 27 08:36:18 2018 -0700 + + [DOCS] Synchronizes captialization in top-level titles (#33605) + +commit 5f19146bace768255363c748fdcaa2c94766f299 +Author: Christoph Büscher +Date: Thu Sep 27 16:27:49 2018 +0200 + + [SQL] Clean up LogicalPlanBuilder#doJoin (#34048) + + Currently the local `type` and `condition` variables are unused and can be removed. + This code can be added later again if joins are supported. + +commit 899a7c7d99b9b3e1f7dece16a614bccaf780bebe +Author: Jason Tedor +Date: Thu Sep 27 09:47:51 2018 -0400 + + Fix remote cluster seeds fallback (#34090) + + Recently we introduced the settings cluster.remote to take the place of + search.remote for configuring remote cluster connections. We made this + change due to the fact that we have generalized the remote cluster + infrastructure to also be used within cross-cluster replication and not + only cross-cluster search. For backwards compatibility, when we made this + change, we allowed that cluster.remote would fallback to + search.remote. Alas, the initial change for this contained a bug for + handling the proxy and seeds settings. The bug for the seeds settings + arose because we were manually iterating over the concrete settings only + for cluster.remote seeds but not for search.remote seeds. This commit + addresses this by iterating over both cluster.remote seeds and + search.remote seeds. Additionally, when checking for existence of proxy + settings, we have to not only check cluster.remote proxy settings, but + also fallback to search.remote proxy settings. This commit addresses + both issues, and adds tests for these situations. + +commit 609ccaad07cbb2a2629c8e051a24e98538bba863 +Author: Dimitris Athanasiou +Date: Thu Sep 27 15:33:22 2018 +0300 + + [ML][HLRC] Replace REST-based ML test cleanup with the ML client (#34109) + + Now that all basic APIs for managing jobs and datafeeds have been + implemented we replace the duplicated `MlRestTestStateCleaner` + with an implementation that uses the HLRC Machine Learning client + itself. + +commit 269ae0bc15e312e91beb497191e44516197b66fa +Author: Jim Ferenczi +Date: Thu Sep 27 14:19:08 2018 +0200 + + Handle MatchNoDocsQuery in span query wrappers (#34106) + + * Handle MatchNoDocsQuery in span query wrappers + + This change adds a new SpanMatchNoDocsQuery query that replaces + MatchNoDocsQuery in the span query wrappers. + The `wildcard` query now returns MatchNoDocsQuery if the target field is not + in the mapping (#34093) so we need the equivalent span query in order to + be able to pass it to other span wrappers. + + Closes #34105 + +commit cb4cdf17f0fc09c5066c8fadc64becf0b5d3cc73 +Author: Christoph Büscher +Date: Thu Sep 27 11:10:45 2018 +0200 + + Update MovAvgIT AwaitsFix bug url + +commit 1cc53b598868337c184601f18db08071791d4290 +Author: Alan Woodward +Date: Thu Sep 27 10:02:29 2018 +0100 + + Bad regex in CORS settings should throw a nicer error (#34035) + + Currently a bad regex in CORS settings throws a PatternSyntaxException, which + then bubbles up through the bootstrap code, meaning users have to parse a + stack trace to work out where the problem is. We should instead catch this + exception and rethrow with a more useful error message. + +commit 53d10bb3b2ab1c575c52a01dc45d94ccda63f094 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Sep 27 18:40:52 2018 +1000 + + [HLRC] Support for role mapper expression dsl (#33745) + + This commit adds support for role mapping expression dsl. + Functionally it is similar to what we have on the server side + except for the rule evaluation which is not required on the client. + + The role mapper expression can either be field expression or + composite expression of one or more expressions. Role mapper + expression parser is used to parse JSON DSL to list of expressions. + + This forms the base for role mapping APIs (get, post/put and delete) + +commit a15b1b97d22d63fd21c888b576bd130db8680af7 +Author: Alexander Reelsen +Date: Thu Sep 27 09:40:15 2018 +0200 + + Watcher: Reduce script cache churn by checking for mustache tags (#33978) + + Watcher is using a lot of so called TextTemplate fields in a watch + definition, which can use mustache to insert the watch id for example. + For the user it is non-obvious which field is just a string field or + which field is a text template. + + This also means, that for every such field, we currently do a script + compilation, even if the field does not contain any mustache syntax. + + This will lead to an increased script cache churn, because those + compiled scripts (that only contain a string), will evict other scripts. + On top of that, this also means that an unneeded compilation has + happened, instead of returning that string immediately. + + The usages of mustache templating are in all of the actions (most of the time far + more than one compilation) as well as most of the inputs. + + Especially when running a lot of watches in parallel, this will reduce + execution times and help reuse of real scripts. + +commit bda7bc145b467e6b4300ca696310532ccfe05fad +Author: Simon Willnauer +Date: Thu Sep 27 09:06:04 2018 +0200 + + Fold EngineSearcher into Engine.Searcher (#34082) + + EngineSearcher can be easily folded into Engine.Searcher which removes + a level of inheritance that is necessary for most of it's subclasses. + This change folds it into Engine.Searcher and removes the dependency on + ReferenceManager. + +commit 0301062c6e177deee66924117ac094a3e49fe77d +Author: Yogesh Gaikwad +Date: Thu Sep 27 15:15:39 2018 +1000 + + Mute SpanMultiTermQueryBuilderTests#testToQuery + +commit acd80a1e0722536fa18220d640ac423155823954 +Author: Armin Braun +Date: Thu Sep 27 06:02:24 2018 +0200 + + TESTS: Enable DEBUG Logging in Flaky Test (#34091) + + * This should surface what errors are thrown on CI + and in org.elasticsearch.transport.RemoteClusterConnection.ConnectHandler#collectRemoteNodes + (the sequence of caught error in the last catch block and moving on to the next seed node + seems to be the only path by which the errors logged in #33756 could come about) + * Relates #33756 + +commit ea9b33527e6d3d1086a401af668249cd679d2dca +Author: Nhat Nguyen +Date: Wed Sep 26 22:38:25 2018 -0400 + + TEST: Add engine is closed as expected failure msg + + This commit adds "engine is closed" as an expected failure message. + This change is due to #33967 in which we might access a closed engine on + promotion. + + Relates #33967 + +commit 12d94e44b8306f315f9e936bc46be55fd867122b +Author: Nhat Nguyen +Date: Wed Sep 26 18:39:10 2018 -0400 + + Adjust bwc version for max_seq_no_of_updates + + Relates #33967 + Relates #33842 + +commit ae8e54493dfb5e14feca6d8fe0ffba7bc045aff8 +Author: Simon Willnauer +Date: Thu Sep 27 00:16:17 2018 +0200 + + Build DocStats from SegmentInfos in ReadOnlyEngine (#34079) + + This change is related to #33903 that ports the DocStats + simplification to the master branch. This change builds the docStats + in the ReadOnlyEngine from the last committed segment infos rather than + the reader. + + Co-authored-by: Tanguy Leroux + +commit 1d08f63effcb387473e2adedae2af79427a47cc0 +Author: Julie Tibshirani +Date: Wed Sep 26 15:08:35 2018 -0700 + + When creating wildcard queries, use MatchNoDocsQuery when the field type doesn't exist. (#34093) + +commit e4c3fd6d9624882abb97ec5b4d67141d80428635 +Author: Lisa Cawley +Date: Wed Sep 26 14:59:31 2018 -0700 + + [DOCS] Moves graph to docs folder (#33472) + +commit 2b730d1b9d7bfdfeee76d7cf8701a2a74ad0fc1c +Author: Simon Willnauer +Date: Wed Sep 26 23:50:31 2018 +0200 + + Mute MovAvgIT#testHoltWintersNotEnoughData + + Relates to #34098 + +commit a48b86e7c632b1c51e7f424b19d5c03780c09cc8 +Author: Jay Modi +Date: Wed Sep 26 14:42:22 2018 -0600 + + Security: use default scroll keepalive (#33639) + + Security previously hardcoded a default scroll keepalive of 10 seconds, + but in some cases this is not enough time as there can be network + issues or overloading of host machines. After this change, security + will now use the default keepalive timeout, which is controllable using + a setting and the default value is 5 minutes. + +commit fcb60acc3481ce648e9c8fc810b00d8b3f5cb81b +Author: Jay Modi +Date: Wed Sep 26 14:27:35 2018 -0600 + + Calculate changed roles on roles.yml reload (#33525) + + In order to optimize the use of the role cache, when the roles.yml file + is reloaded we now calculate the names of removed, changed, and added + roles so that they may be passed to any listeners. This allows a + listener to selectively clear cache for only the roles that have been + modified. The CompositeRolesStore has been adapted to do exactly that + so that we limit the need to reload roles from sources such as the + native roles stores or external role providers. + + See #33205 + +commit 85e4ef3429ae2aa3b4ff07a7ceb9ba07bc921756 +Author: Ryan Ernst +Date: Wed Sep 26 13:22:48 2018 -0700 + + Scripting: Reflect factory signatures in painless classloader (#34088) + + It is sometimes desirable to pass a class into a script constructor that + will not actually be exposed in the script whitelist. This commit uses + reflection when creating the compiler to find all the classes of the + factory method signature, and make the classloader that wraps lookup + also expose these classes. + +commit 80c5d30f30c311544844187d412498990ba5ed5a +Author: Mayya Sharipova +Date: Wed Sep 26 14:24:31 2018 -0400 + + XContentBuilder to handle BigInteger and BigDecimal (#32888) + + Although we allow to index BigInteger and BigDecimal into a keyword + field, source filtering on these fields would fail + as XContentBuilder was not able to deserialize BigInteger and BigDecimal + to json. + + This modifies XContentBuilder to allow to handle BigInteger and + BigDecimal. + + Closes #32395 + +commit de8bfb908f6fe1706e66bdf0d2e65ac1539cab59 +Author: Julie Tibshirani +Date: Wed Sep 26 09:36:41 2018 -0700 + + Delegate wildcard query creation to MappedFieldType. (#34062) + + * Delegate wildcard query creation to MappedFieldType. + * Disallow wildcard queries on collation fields. + * Disallow wildcard queries on non-string fields. + +commit 4fbe84edf6a15f3c6409f6fd3104e40cbb90543c +Author: Jack Conradson +Date: Wed Sep 26 09:33:08 2018 -0700 + + Painless: Cleanup Cache (#33963) + + * Adds equals/hashcode methods to the Painless* objects within the lookup package. + * Changes the caches to use the Painless* objects as keys as well as values. This forces + future changes to taken into account the appropriate values for caching. + * Deletes the existing caching objects in favor of Painless* objects. This removes a pair of + bugs that were not taking into account subtle corner cases related to augmented methods + and caching. + * Uses the Painless* objects to check for equivalency in existing Painless* objects that + may have already been added to a PainlessClass. This removes a bug related to return + not being appropriately checked when adding methods. + * Cleans up several error messages. + +commit ddce9704d4cbd077b59a24b5608f8a1a1d84ee82 +Author: Nik Everett +Date: Wed Sep 26 11:20:52 2018 -0400 + + Logging: Drop two deprecated methods (#34055) + + This drops two deprecated methods from `ESLoggerFactory`, switching all + calls to those methods to calls to methods of the same name on + `LogManager`. + +commit 3c2841d493789ceadac2420fce451798d83d09e6 +Author: Adrien Grand +Date: Wed Sep 26 17:11:37 2018 +0200 + + REST test for typeless APIs. (#33934) + + This commit duplicates REST tests for the + - `indices.create` + - `indices.put_mapping` + - `indices.get_mapping` + - `index` + - `get` + - `delete` + - `update` + - `bulk` + APIs, so that we both test them when used without types (include_type_name=false) + and with types, mostly for mixed-version cluster tests. + + Given a suite called `X_test_name.yml`, I first copied it to + `(X+1)_test_name_with_types.yml` and then changed `X_test_name.yml` to set + `include_type_name=false` on every API that supports it. + + Relates #15613 + +commit 7800b4fa9188255baa114496e2941c6c627ccad9 +Author: Ryan Ernst +Date: Wed Sep 26 07:56:25 2018 -0700 + + Core: Abstract DateMathParser in an interface (#33905) + + This commits creates a DateMathParser interface, which is already + implemented for both joda and java time. While currently the java time + DateMathParser is not used, this change will allow a followup which will + create a DateMathParser from a DateFormatter, so the caller does not + need to know the internals of the DateFormatter they have. + +commit ff2bbdf765a40a940c6c1385457575b84c10cd49 +Author: Shivaank121 +Date: Wed Sep 26 20:03:58 2018 +0530 + + [Docs] Correct typo in configuring-es.asciidoc (#34064) + +commit 25d74bd0cb91eef7aa19b02522f2259227a0738a +Author: Zachary Tong +Date: Wed Sep 26 10:09:31 2018 -0400 + + Prefer mapped aggs to lead reductions (#33528) + + Previously, unmapped aggs try to delegate reduction to a sibling agg that is + mapped. That delegated agg will run the reductions, and also + reduce any pipeline aggs. But because delegation comes before running + pipelines, the unmapped agg _also_ tries to run pipeline aggs. + + This causes the pipeline to run twice, and potentially double it's output + in buckets which can create invalid JSON (e.g. same key multiple times) + and break when converting to maps. + + This fixes by sorting the list of aggregations ahead of time so that mapped + aggs appear first, meaning they preferentially lead the reduction. If all aggs + are unmapped, the first unmapped agg simply creates a new unmapped object + and returns that for the reduction. + + This means that unmapped aggs no longer defer and there is no chance for + a secondary execution of pipelines (or other side effects caused by deferring + execution). + + Closes #33514 + +commit 1871e7f7e96a3510a89a249d5398b7eed6ba8dd6 +Author: Nik Everett +Date: Wed Sep 26 09:48:15 2018 -0400 + + Search: Simply SingleFieldsVisitor (#34052) + + `SingleFieldsVisitor` is meant to load a single stored field but it + manages to be quite complex to reason about because it inherits from our + "basic" `FieldsVisitor` which is designed to load many fields. This + breaks that inheritance and adds logic to `SingleFieldsVisitor` so it can + be properly stand alone. While this amounts to more lines of code they + ought to be significantly easier to reason about. + +commit e6d1af66caad30e0c1066682f3edcd903ad56ed2 +Author: Benjamin Trent +Date: Wed Sep 26 06:35:17 2018 -0700 + + Changing bucket length and data indexed timestamps (#33995) + + * Changing bucket length and data indexed timestamps + + * changing test variable names to lessen confusion + +commit 1413ace74f82ad22ec9f2ad74645faa85d5a2f25 +Author: David Roberts +Date: Wed Sep 26 14:02:14 2018 +0100 + + Mute testSplitFromOneToN and testCreateShrinkIndexToN on Windows + + Relates #34080 + +commit ba3ceeaccf9b59ad16af60e279dbd6339e091a48 +Author: Christoph Büscher +Date: Wed Sep 26 14:09:32 2018 +0200 + + Clean up "unused variable" warnings (#31876) + + This change cleans up "unused variable" warnings. There are several cases were we + most likely want to suppress the warnings (especially in the client documentation test + where the snippets contain many unused variables). In a lot of cases the unused + variables can just be deleted though. + +commit 48c169e065d78e198f5df7a47172c1e76ecfe416 +Author: Nhat Nguyen +Date: Wed Sep 26 08:00:10 2018 -0400 + + CCR: replicates max seq_no of updates to follower (#34051) + + This commit replicates the max_seq_no_of_updates on the leading index + to the primaries of the following index via ShardFollowNodeTask. The + max_seq_of_updates is then transmitted to the replicas of the follower + via replication requests (that's BulkShardOperationsRequest). + + Relates #33656 + +commit d995fc85c60a639d827b45263cf4e29fba4f2d08 +Author: David Turner +Date: Wed Sep 26 12:18:13 2018 +0100 + + Integrate LeaderChecker with Coordinator (#34049) + + This change ensures that follower nodes periodically check that their leader is + healthy, and that they elect a new leader if not. + +commit 7bf216f4a17b297850a0cffe71b1bf2b8b79bdbe +Author: Christoph Büscher +Date: Wed Sep 26 11:59:36 2018 +0200 + + [Docs] Add reason to use Settings API over config file (#32405) + + Extending the warning to prefer the `setttings` API over changes in the + configuration file by giving reasons why this might be dangerous. + +commit eae548747734d1d5cd9d6b6cd9f100c66a1bbc84 +Author: Martijn van Groningen +Date: Wed Sep 26 09:31:36 2018 +0200 + + [CCR] set minimum version to 6.5.0 + +commit a2558804974687541f794d95a7bc53ec3a8caa17 +Author: Jim Ferenczi +Date: Wed Sep 26 08:59:41 2018 +0200 + + Add nested and object fields to field capabilities response (#33803) + + This commit adds nested and object fields to the field capabilities response. + + Closes #33237 + +commit 96b341798552562b0a3a4309467d73a59126212b +Author: Martijn van Groningen +Date: Wed Sep 26 07:34:51 2018 +0200 + + [CCR] Don't auto follow follow indices in the same cluster. (#33944) + +commit be8475955ec96a7d8551314e0eb2580bd5645b2f +Author: Ryan Ernst +Date: Tue Sep 25 22:08:02 2018 -0700 + + Scripting: Use ParameterMap for deprecated ctx var in update scripts (#34065) + + This commit removes the sysprop controlling whether ctx is in params for + update scripts and replaces it with use of the new ParameterMap, which + outputs a deprecation warning whenever params.ctx is used. + +commit 8a56369f5bbcbc2c66b01e2e888a2bd904f46081 +Author: Nhat Nguyen +Date: Tue Sep 25 19:20:00 2018 -0400 + + Move max_unsafe_auto_id_timestamp constant to Engine (#34025) + + We should not access InternalEngine in other classes. + +commit 0f878eff19cae6dcef0c61692e54ba88a209ed2c +Author: Jim Ferenczi +Date: Tue Sep 25 21:38:47 2018 +0200 + + Add a limit for graph phrase query expansion (#34031) + + Today query parsers throw TooManyClauses exception when a query creates + too many clauses. However graph phrase queries do not respect this limit. + This change adds a protection against crazy expansions that can happen when + building a graph phrase query. This is a temporary copy of the fix available + in https://issues.apache.org/jira/browse/LUCENE-8479 but not merged yet. + This logic will be removed when we integrate the Lucene patch in a future + release. + +commit 1e6780d703f258cd180a0b9834e76811c0002d6a +Author: Igor Motov +Date: Tue Sep 25 14:16:47 2018 -0400 + + Mute AckClusterUpdateSettingsIT + + Tracked by #33673 + +commit 0ba18557406b5fbe6266bee8f1b05afb1f11c72a +Author: Armin Braun +Date: Tue Sep 25 19:29:22 2018 +0200 + + INGEST: Tests for Drop Processor (#33430) + + * INGEST: Tests for Drop Processor + + * UT for behavior of dropped callback + and drop processor + * Moved drop processor to `server` + project to enable this test + * Simple IT + * Relates #32278 + +commit 5840be6a6b581f90581faa663daa95c8e035f411 +Author: Marios Trivyzas +Date: Tue Sep 25 19:20:25 2018 +0200 + + SQL: Prevent StackOverflowError when parsing large statements (#33902) + + Implement circuit breaker logic in the parser which catches expressions + that can blow up the tree and result in StackOverflowError being thrown. + + Co-authored-by: Costin Leau + +commit c0bfc07f5338672ea59fe1b873d985b212b5be1e +Author: Gordon Brown +Date: Tue Sep 25 10:16:01 2018 -0600 + + Only make indexes read-only on Shrink and ForceMerge actions (#33907) + + ILM now only forces indices to become read only in the case of Shrink + and Force Merge actions, as these are most useful in cases where the + index is no longer being written to. + +commit cc70352b3fbc567f3c43683fcb9371fee61e788a +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Tue Sep 25 15:30:10 2018 +0100 + + [ML] Modify thresholds for normalization triggers (#33663) + + [ML] Modify thresholds for normalization triggers + + The (arbitrary) threshold factors used to judge if scores have + changed significantly enough to trigger a look-back renormalization have + been changed to values that reduce the frequency of such + renormalizations. + + Added a clause to treat changes in scores as a 'big change' if it would + result in a change of severity reported in the UI. + + Also altered the clause affecting small scores so that a change should + be considered big if scores have changed by at least 1.5. + + Relates https://github.com/elastic/machine-learning-qa/issues/263 + +commit c01b6ffb80bfe6ef22875262d9f3c72fd6bed81f +Author: Nik Everett +Date: Tue Sep 25 10:15:43 2018 -0400 + + CRUD: Fix wait for refresh tests (#33973) + + When we implemented `refresh=wait_for` I added a test with the wrong + name. This caused us to not run it. The test asserted that running + several operations with `refresh=wait_for` did not fail if the index was + `_close`d while the operations were waiting. But to be honest, failure + here isn't that bad. The index being waited on is closed. You can't do + anything with it any way. The most important thing is actually that + these operations don't hang forever. Because hanging forever means that + the resources used by the operations aren't freed. + + Anyway, when I noticed the error I reenabled the test. But they don't + pass consistently because *sometimes* the operations being tested fail. + They don't seem to hang and they always fail with "this index is closed + so you can't do anything with it" sorts of messages. + + When the test started failing we disabled it again. This reenables the + test but causes it to ignore these "index is closed" failures. We'd + prefer they not happen at all but in the grand scheme of things they are + fine and making sure these operations don't hang is much more important. + + This also updates the test to bring it more in line with my current + understanding of the "right" way to use the low level rest client. + +commit 16956a1a0530cc37aa68ff0e59d9c7e067357d82 +Author: Nikolay Vasiliev +Date: Tue Sep 25 15:32:27 2018 +0200 + + [DOCS] Clarify 'type' parameter meaning for custom analyzer (#34012) + + This pull request improves the docs on the meaning of type parameter on the custom + analyzer doc page. + + Closes #33456 + +commit ecc087a5bb8b234916e69e12d1337e15f87fde5e +Author: Christoph Büscher +Date: Tue Sep 25 15:25:54 2018 +0200 + + Remove Join utility class (#34037) + + The functionality can be replaces with String.join in new Java versions. + +commit f886eebd991871976cd9519bda00041991b95b63 +Author: David Turner +Date: Tue Sep 25 14:04:22 2018 +0100 + + Fix CoordinatorTests some more (#34039) + + Today the `CoordinatorTests` are not completely reliable. These changes make + them more so, by removing a couple of assertions that we do not expect to pass + (yet). + +commit 7c63f5455b157df32a15f8a17ee9306b50bff916 +Author: David Turner +Date: Tue Sep 25 13:58:05 2018 +0100 + + Use a threadsafe map in SearchAsyncActionTests (#33700) + + Today `SearchAsyncActionTests#testFanOutAndCollect` uses a simple `HashMap` for + the `nodeToContextMap` variable, which is then accessed from multiple threads + without, apparently, explicit synchronisation. This provides an explanation for + the test failure identified in #29242 in which `.toString()` returns `"[]"` + just before `.isEmpty` returns `false`, without any concurrent modifications. + + This change converts `nodeToContextMap` to a `newConcurrentMap()` so that this + cannot occur. It also fixes a race condition in the detection of double-calling + the subsequent search phase. + + Closes #29242. + +commit 5166dd0a4c6b53560cb873dbda1f41099e1554fa +Author: Nhat Nguyen +Date: Tue Sep 25 08:07:57 2018 -0400 + + Replicate max seq_no of updates to replicas (#33967) + + We start tracking max seq_no_of_updates on the primary in #33842. This + commit replicates that value from a primary to its replicas in replication + requests or the translog phase of peer-recovery. + + With this change, we guarantee that the value of max seq_no_of_updates + on a replica when any index/delete operation is performed at least the + max_seq_no_of_updates on the primary when that operation was executed. + + Relates #33656 + +commit 970407c6632ab8330761c6f7e202d5ae9e9f2a35 +Author: Luca Cavanna +Date: Tue Sep 25 14:03:07 2018 +0200 + + [DOCS] add comment to clarify cluster name resolution (#34014) + + We currently fallback to local indices whenever a remote cluster is not found, as there may still be indices / aliases with the same name. Such behaviour is lenient but needs to be kept for backwards compatibility. Clarified that in the code so we don't forget. + + Relates to #26247 + +commit dfe5af04115063f564255a3509e308d50caaebdd +Author: David Roberts +Date: Tue Sep 25 12:52:51 2018 +0100 + + [ML] Return both Joda and Java formats from structure finder (#33900) + + Previously the timestamp_formats field in the response + from the find_file_structure endpoint contained Joda + timestamp formats. This change makes that clear by + renaming the field to joda_timestamp_formats, and also + adds a java_timestamp_formats field containing the + equivalent Java time format strings. + +commit 612201aee0521f0340f0cba43678ef454437b3e0 +Author: Adrien Grand +Date: Tue Sep 25 13:48:25 2018 +0200 + + Fix created version for similarity validation. (#33890) + + It mistakenly uses the Elasticsearch major version instead of the Lucene major + version. I noticed it when backporting, it is not noticeable on master because + the only two Lucene versions that are supported, 7 and 8, encode norms the same + way, unlike Lucene 6. + +commit 2ca177be413c9e275eed640c32fcd9617d6e7749 +Author: Marios Trivyzas +Date: Tue Sep 25 12:17:14 2018 +0200 + + SQL: Fix query translation of GroupBy with Having (#34010) + + Two issues are resolved: + 1. The `value_type` should be either long or double in case of numeric. + 2. The key label for the aggregate filter (having) was duplicate of an aggr key label. + + Fixes: #33520 + +commit 7bc7624caf62e20a5d0864d4df26b63bb4ce0584 +Author: David Kyle +Date: Tue Sep 25 10:18:30 2018 +0100 + + Leniently parse ml persistent task parameters (#33950) + +commit 02fb5aa4ec6c1451c460e9915c927c92be0ab7e1 +Author: Daniel Mitterdorfer +Date: Tue Sep 25 10:13:23 2018 +0200 + + Remove leftover doc about format being updatable + + With this commit we remove a leftover in the docs about the `format` + field being updatable. This is not true since we removed support for + updates in #25285. + + Closes #33986 + Relates #25285 + Relates #34006 + +commit 679fb698d0beb1f7b83658e0bc0b6293b2dec668 +Author: Yannick Welsch +Date: Tue Sep 25 09:44:35 2018 +0200 + + Zen2: Trigger join when active master detected (#34008) + + Triggers a join when an active master is detected. In order to avoid spamming joins, deduplicates join request based on pair. This ensures that a new join is sent whenever the term is incremented or when a new master is found. + + Also changes the logging of join failures from DEBUG to INFO. These join failures should be happening rarely, and can either indicate a failed election (which should be rare) or a configuration issue. + +commit 1d47c9582bbb5839a1e954041e003dce143a4928 +Author: David Turner +Date: Tue Sep 25 08:43:47 2018 +0100 + + Fix CoordinatorTests (#34002) + + Today the CoordinatorTests are not very reliable if two elections are scheduled + concurrently. Although we expect occasional failures due to this, in fact the + failures are much more common than expected due to a handful of issues. This PR + fixes these issues. + +commit bf6cf6b6d9e45ba5c27c293191b84ee21c2e6bbd +Author: Hendrik Muhs +Date: Tue Sep 25 09:15:52 2018 +0200 + + refactor CompositeValuesSourceParserHelper for reusage by making it public (#33945) + + refactor CompositeValuesSourceParserHelper for reusage by making it public and moving toXContent into it + +commit 3af8fc74c789d2dec3fae8254aec738bfa2b5c98 +Author: David Turner +Date: Tue Sep 25 07:50:18 2018 +0100 + + Make TransportService more test-friendly (#33869) + + Today, TransportService uses System.currentTimeMillis() to get the current time + to report on things like timeouts, and enqueues lambdas for future execution. + However, in tests it is useful to be able to fake out the current time and to + see what all these enqueued lambdas are really for. This change alters the + situation so that we can obtain the time from the more easily-faked + ThreadPool#relativeTimeInMillis(), and implements some friendlier toString() + methods on the various Runnables so we can see what they are later. + +commit 2ad06f6e670611f0a5c00e816adcda4c492c7805 +Author: Armin Braun +Date: Tue Sep 25 08:46:38 2018 +0200 + + NETWORKING: Upgrade to Netty 4.1.29 (#33984) + +commit 793b2a94b451e65730a4b57f322db87d5c285d35 +Author: Martijn van Groningen +Date: Tue Sep 25 07:19:46 2018 +0200 + + [CCR] Expose auto follow stats to monitoring (#33886) + +commit 9b86e9aed85efd17ac516ffed7e452c3232e855a +Author: Shaunak Kashyap +Date: Mon Sep 24 18:52:26 2018 -0700 + + [Monitoring] Add cluster metadata to cluster_stats docs (#33860) + + * WIP + + * Adding cluster metadata to cluster stats monitoring doc + + * Fixing checkstyle errors + + * Adding missing license header + + * Updating tests + + * Getting cluster settings from cluster state + + * Removing more unnecessary changes + + * Adding cluster metadata settings to cluster_stats docs + + * Updating test to include cluster metadata + + * Fixing checkstyle + + * Guarding against NPE + + * Updating test fixture + +commit e6cb9d642b262c347f31e84500cb2dc1309f42f1 +Author: Julie Tibshirani +Date: Mon Sep 24 18:06:35 2018 -0700 + + Fix a broken reference to types removal in the Java API docs. + +commit 0fae6d39f54a9adc335d3779af72ddd3a0ff5ad4 +Author: Andrei Stefan +Date: Tue Sep 25 02:42:18 2018 +0300 + + SQL: functions docs update (#34000) + + * Changed the format of the String functions documentation page. + * Adopted the same format for Math functions, but completely changed the examples. + * Added missing documentation for Math functions. + +commit 704d3e4c245fe9f92b5e73338c3ea33d98374398 +Author: Julie Tibshirani +Date: Mon Sep 24 16:30:38 2018 -0700 + + Add a deprecation warning in the type query documentation. (#34017) + +commit 6ec36b127367acb4e7a0046024bd75583dbc6f43 +Author: Nhat Nguyen +Date: Mon Sep 24 17:47:10 2018 -0400 + + CCR: Make AutoFollowMetadata immutable (#33977) + + We should make AutoFollowMetadata immutable to avoid being inconsistent + when one thread modifies it while other reads it. + +commit 018714f93867cfcca7d0f115816783b3b2c10ba3 +Author: Armin Braun +Date: Mon Sep 24 23:40:29 2018 +0200 + + TESTS: Stabilize Renegotiation Test (#33943) + + * TESTS: Stabilize Renegotiation Test + + * The second `startHandshake` is not synchronous and a read of only + 50ms may fail to trigger it entirely (the failure can be reproduced reliably by setting the socket timeout to `1`) + => fixed by retrying the read until the handshake finishes (a longer timeout would've worked too, + but retrying seemed more stable) + * Closes #33772 + +commit 74d7be805a2546c55749b3f971f1a1c8b392d32d +Author: Benjamin Trent +Date: Mon Sep 24 12:54:32 2018 -0700 + + Make certain ML node settings dynamic (#33565) (#33961) + + * Make certain ML node settings dynamic (#33565) + + * Changing to pull in updating settings and pass to constructor + + * adding note about only newly opened jobs getting updated value + +commit 02b483c372683a0abfde2f7c398463e58d181581 +Author: David Turner +Date: Mon Sep 24 20:07:32 2018 +0100 + + Logging improvements in CoordinatorTests (#33991) + + Today, we know that CoordinatorTests sometimes fail to stabilise due to an + election collision. This change improves the logging that occurs when an + election collision occurs so it will be easier to see if this is happening when + analysing a test failure. We also wrap the call to + masterService.submitStateUpdateTask() in a context that logs the node on which + it runs. + + We also introduce the InitialJoinAccumulator instead of using a placeholder + CandidateJoinAccumulator at startup, which reduces the cases to consider in + CandidateJoinAccumulator.close() and tightens up the assertions we can make + here. + +commit 2795ef561f527af5fe3aaf0f6fc7cb70bd70d54e +Author: Martijn van Groningen +Date: Mon Sep 24 20:26:13 2018 +0200 + + [CCR] Add get auto follow pattern api (#33849) + + Relates to #33007 + +commit 1c579646d3b40bd0bfc3cf9c146da27b4f964c3b +Author: debadair +Date: Mon Sep 24 10:29:12 2018 -0700 + + [DOCS] Removed Homebrew installation section (#34011) + +commit 243e863f6e862074b564f909e90a8fd83ae8ad01 +Merge: 90de436e55f 25bc8c4b5a0 +Author: Lee Hinman +Date: Mon Sep 24 10:33:51 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 25bc8c4b5a0394cb1116361ff1d0fa432ed7053e +Author: Armin Braun +Date: Mon Sep 24 17:57:27 2018 +0200 + + Fix typo `NodeEnvironment#assertPathsDoNotExist` (#33996) + + * We want to check the individual paths here one by one to + get a better to interpret assertion message + +commit 8e8bd56cc7b0191ab48b7a39ae8152ee29b63406 +Author: Julie Tibshirani +Date: Mon Sep 24 08:56:13 2018 -0700 + + In MatchQuery, remove a check for fragile search analyzers. (#33927) + + As far as I can tell this guard against fragile analyzers is no longer relevant, since + we stopped setting special analyzers on numeric fields (3bf6f4). Instead of removing + the guard completely, I opted to keep a check for untokenized + unnormalized fields + to avoid going through the analysis process unnecessarily. + + My motivation for simplifying this check is that I'd like to add support for + `split_queries_on_whitespace` to the new 'queryable object' fields. As it stands, I would + have to add a dedicated instanceof check for the new mapper, which is not optimal. + +commit 2e774e146da485c905846793a2a30ff9666f4781 +Author: Yannick Welsch +Date: Mon Sep 24 17:47:15 2018 +0200 + + Zen2: Update PeerFinder term on term bump (#33992) + + Ensures that the PeerFinder always uses the correct term. + +commit 78e483e8d8de03e4288076a92f4388cbf748129c +Author: Tim Brooks +Date: Mon Sep 24 09:44:44 2018 -0600 + + Introduce abstract security transport testcase (#33878) + + This commit introduces an AbstractSimpleSecurityTransportTestCase for + security transports. This classes provides transport tests that are + specific for security transports. Additionally, it fixes the tests referenced in + #33285. + +commit df333ca305107d7c3f19782f418d526f7d63c2c3 +Author: Ignacio Vera +Date: Mon Sep 24 17:36:48 2018 +0200 + + TESTS: Make score Float#NaN when there is no max score (#33997) + + * TESTS: Make score Float#NaN when there is no max score + + Fixes test failure due to maxScore set to Float#MinValue instead + on Float#NaN. In addition the initial value for maxScore is set to + Float#NEGATIVE_INFINITY so it is an illegal value. + + Closes #33993 + +commit 27ee5e87e4d5cd18306601111be946480a3ccde4 +Author: markharwood +Date: Mon Sep 24 14:19:17 2018 +0100 + + Fix for test version following backport of annotated_text plugin (#33979) + +commit e389d9e2969628278de43f28b805f9166d7bec64 +Author: Luca Cavanna +Date: Mon Sep 24 11:45:33 2018 +0200 + + Clarify RemoteClusterService#groupIndices behaviour (#33899) + + When executing a cross-cluster search, we need to search against all local indices (and no remote indices) in case no indices are specified. Also, if only remote indices are specified, no local indices will be queried. We previously added empty local indices whenever they were not present in the map of the grouped indices, then we would act differently later based on the extracted remote indices. Instead, we now add the empty array for local indices only in case we need to search all local indices; the entry for local indices is not added when local indices should not be searched. This way the grouped indices reflect reality and provide a better indication of what indices will be searched. + +commit 47ed6c79eea40ee3fcdb29a265c21fcd47337df8 +Author: Christophe Bismuth +Date: Mon Sep 24 11:21:47 2018 +0200 + + [TEST] Add validate query tests for empty and malformed queries (#33862) + + Relates to #33095 + +commit a9ded8f1d5f3d8f27a9386c51073c95658666144 +Author: Christoph Büscher +Date: Mon Sep 24 11:03:28 2018 +0200 + + Mute MachineLearningIT#testStartDatafeed + +commit 5a53193db0b00722aefbe49a650b7b40a9684fda +Author: Daniel Mitterdorfer +Date: Mon Sep 24 10:27:43 2018 +0200 + + Remove public modifier in SourceOnlySnapshotTests + + The public modifier is reported as redundant by checkstyle. With this + commit we remove it to avoid the checkstyle error. + +commit 83877ea694fdaad898577fd523f86072cf8a2562 +Author: horacimacias +Date: Mon Sep 24 10:13:35 2018 +0200 + + Fix content type in Watcher docs + + With this commit we change the incorrect content type `image.png` to `image/png` + in an example snippet in the Watcher docs. + + Relates #33955 + +commit 738d9f1a3a4859b8fd079ec1a03a8a0a49b27554 +Author: Simon Willnauer +Date: Mon Sep 24 08:32:32 2018 +0200 + + Drop all fully deleted segments for comparison + + The source only snapshot drops fully deleted segments before snapshotting + them. In order to compare them we need to drop all fully deleted segments + in the test as well. + + Closes #33755 + +commit 7d703c2f92ba784de47ea00959ba7626597e4e94 +Author: Simon Willnauer +Date: Mon Sep 24 07:45:50 2018 +0200 + + Fix AutoQueueAdjustingExecutorBuilder settings validation (#33922) + + Settings validation in AutoQueueAdjustingExecutorBuilder always checked against + a default value which means that we never can change a max queue size that is lower + than the default. This change adds tests and fixes this validation. + +commit ddd5ce5740b9ee067e19da4b16a0baf3438420cc +Author: Nhat Nguyen +Date: Sun Sep 23 22:28:41 2018 -0400 + + TEST: Avoid invalid ranges in ShardChangesActionTests (#33976) + + If numWrites is between 2 and 9, we will issue an invalid range because + the from_seq_no is negative. This commit makes sure that numWrites is at + least 10, and adds an explicit test to verify invalid request ranges. + +commit df521c1d3e46681882d7dd967b5ff0b5e080f822 +Author: Tim Vernum +Date: Mon Sep 24 10:37:22 2018 +1000 + + [TEST] Increase processing delay in LDAP tests (#33410) + + SearchGroupsResolverInMemoryTests was (rarely) fail in a way that + suggests that the server-side delay (100ms) was not enough to trigger + the client-side timeout (5ms). + + The server side delay has been increased to try and overcome this. + + Resolves: #32913 + +commit 432e61c971af77fdacf21c1bc6c2d4677b416398 +Author: Nhat Nguyen +Date: Sat Sep 22 19:29:18 2018 -0400 + + Adjust bwc for resync request (#33964) + + Relates #33964 + +commit f2f08dd6c5a69a4db69a5e78f73021204c0a9e9f +Author: Nhat Nguyen +Date: Sat Sep 22 19:28:20 2018 -0400 + + Adjust bwc for recovery request (#33693) + + Relates #33693 + +commit e7ae2f9d3621edb265d56cc5c8479af2a6ccb3d1 +Author: Nhat Nguyen +Date: Sat Sep 22 11:40:10 2018 -0400 + + Propagate auto_id_timestamp in primary-replica resync (#33964) + + A follow-up of #33693 to propagate max_seen_auto_id_timestamp in a + primary-replica resync. + + Relates #33693 + +commit b89551c4524f6c1b9494157330cded454d391228 +Author: David Roberts +Date: Sat Sep 22 15:48:59 2018 +0100 + + [ML] Display integers without .0 in file structure field stats (#33947) + + Previously numeric values in the field_stats created by the + find_file_structure endpoint were always output with a + decimal point. This looked unfriendly and unnatural for + fields that clearly store integer values. This change + converts integer values to type Integer before output in + the file structure field stats. + +commit 7944a0cb2526188295820461f75269aed435041f +Author: Nhat Nguyen +Date: Sat Sep 22 08:02:57 2018 -0400 + + Track max seq_no of updates or deletes on primary (#33842) + + This PR is the first step to use seq_no to optimize indexing operations. + The idea is to track the max seq_no of either update or delete ops on a + primary, and transfer this information to replicas, and replicas use it + to optimize indexing plan for index operations (with assigned seq_no). + + The max_seq_no_of_updates on primary is initialized once when a primary + finishes its local recovery or peer recovery in relocation or being + promoted. After that, the max_seq_no_of_updates is only advanced internally + inside an engine when processing update or delete operations. + + Relates #33656 + +commit 1761b6c85ca6781903af8c2754133c7426f0eaa0 +Author: David Turner +Date: Sat Sep 22 11:34:16 2018 +0100 + + Introduce FollowersChecker (#33917) + + It is important that the leader periodically checks that its followers are + still healthy and can remain part of its cluster. If these checks fail + repeatedly then the leader should remove the faulty node from the cluster. The + FollowerChecker, introduced in this commit, performs these periodic checks and + deals with retries. + +commit 477391d751e5aaadc54884a46a5ebb86fd33a53a +Author: Vladimir Dolzhenko +Date: Sat Sep 22 10:21:36 2018 +0200 + + Don't test corruption detection within CFS checksum (#33911) + + + Closes #33881 + +commit 17605bf1c69c6634ce66407ac80fc29d7b4b0d10 +Author: Costin Leau +Date: Sat Sep 22 11:03:38 2018 +0300 + + SQL: Move away internally from JDBCType to SQLType (#33913) + + Replace JDBCType enum with SQLType interface for extensibility + + Fix #33094 + +commit ea3f3e4b611da73ce39702ef4c99d0b19d9d0194 +Author: Benjamin Trent +Date: Fri Sep 21 12:13:35 2018 -0700 + + HLRC: ML Stop datafeed API (#33946) + + * HLRC: ML stop data feed api + +commit a612dd12724f3caace58a2f3ac4f0b7e0508e765 +Author: Yannick Welsch +Date: Fri Sep 21 18:40:12 2018 +0200 + + Zen2: Add node id to log output of CoordinatorTests (#33929) + + With recent changes to the logging framework, the node name can no longer be injected into the logging output using the node.name setting, which means that for the CoordinatorTests (which are simulating a cluster in a fully deterministic fashion using a single thread), as all the different nodes are running under the same test thread, we are not able to distinguish which log lines are coming from which node. This commit readds logging for node ids in the CoordinatorTests, making two very small changes to DeterministicTaskQueue and TestThreadInfoPatternConverter. + +commit 9e1757c5cd6a773c108d0dba12c675a7775f3907 +Author: Lee Hinman +Date: Fri Sep 21 10:31:32 2018 -0600 + + Add commented out JVM options for G1GC (#33685) + + * Add commented out JVM options for G1GC + + These options are available now that we will be supporting G1GC for Java 10 and + above. They are also designed so that the CMS options don't have to be commented + out in order for the G1 options to take effect. + + * Update wording + +commit e1e5f4072715e513c88d16eacc848f8cfab624d5 +Author: Martijn van Groningen +Date: Fri Sep 21 18:08:29 2018 +0200 + + [CCR] Move headers from auto follow pattern to auto follow metadata (#33846) + + This ensures that we will not serialize the headers as part of the + auto follow pattern in the to be added get auto follow api. + +commit b48d5a89427503aa525af4fc4d1b2f95ba22c23a +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Fri Sep 21 17:26:55 2018 +0200 + + [TEST] ClientYamlSuiteRestApiParser to parse spec without path parts (#33720) + + Previously ClientYamlSuiteRestApiParser threw an exception when an api + spec contained neither path parts nor url parameter sections. + + Closes #31649 + +commit bf0a0f74dae85e2ca7fe6c07a3ab585b7069a294 +Author: Benjamin Trent +Date: Fri Sep 21 05:56:38 2018 -0700 + + HLRC: ML start data feed API (#33898) + + * HLRC: ML start data feed API + +commit 1de2a925ce8bf30d53ad71dfcd6e33ebbf7827c6 +Author: Alexander Reelsen +Date: Fri Sep 21 14:22:34 2018 +0200 + + Watcher: Ensure that execution triggers properly on initial setup (#33360) + + This commit reverts most of #33157 as it introduces another race + condition and breaks a common case of watcher, when the first watch is + added to the system and the index does not exist yet. + + This means, that the index will be created, which triggers a reload, but + during this time the put watch operation that triggered this is not yet + indexed, so that both processes finish roughly add the same time and + should not overwrite each other but act complementary. + + This commit reverts the logic of cleaning out the ticker engine watches + on start up, as this is done already when the execution is paused - + which also gets paused on the cluster state listener again, as we can be + sure here, that the watches index has not yet been created. + + This also adds a new test, that starts a one node cluster and emulates + the case of a non existing watches index and a watch being added, which + should result in proper execution. + + Closes #33320 + +commit 9c0316869bba93786333a16334873bb36c873096 +Author: Vladimir Dolzhenko +Date: Fri Sep 21 14:00:02 2018 +0200 + + Store: keep IndexFormatTooOldException and IndexFormatTooNewException in corruption marker (#33920) + + Closes #33916 + +commit cac93949feaad2c1e7c0c2f229b3f792d04e6432 +Author: Nik Everett +Date: Fri Sep 21 07:55:50 2018 -0400 + + API: Drop deprecated methods from Retry (#33925) + + We deprecated the `Retry.withBackoff` flavors with `Settings` in 6.5 + because they were no longer needed. This drops them form 7.0. + +commit e17bd8e9139bd4f6b8d648cc21dec597a44dbaf2 +Author: Benjamin Trent +Date: Fri Sep 21 04:49:20 2018 -0700 + + Removing poor randomization for node name (#33918) + +commit b654d986d7814b66808e40dfe8a4a891b7d6a7f4 +Author: Christoph Büscher +Date: Fri Sep 21 11:52:31 2018 +0200 + + Add OneStatementPerLineCheck to Checkstyle rules (#33682) + + This change adds the OneStatementPerLineCheck to our checkstyle precommit + checks. This rule restricts the number of statements per line to one. The + resoning behind this is that it is very difficult to read multiple statements on + one line. People seem to mostly use it in short lambdas and switch statements in + our code base, but just going through the changes already uncovered some actual + problems in randomization in test code, so I think its worth it. + +commit 3a5b8a71b4f7d5473a7bfbce2f2eb5a6bfefdc17 +Author: Armin Braun +Date: Fri Sep 21 10:08:16 2018 +0200 + + NETWORKING: Fix Portability of SO_LINGER=0 in Tests (#33895) + + * Setting SO_LINGER for open but not connected non-blocking sockets + throws on OSX + * Fixed by only applying setting to connected sockets which will save + the same number of FDs as doing it on open sockets anyway + * closes #33879 + +commit 5f7f793f433f293593d566dd63178e6f9dae3155 +Author: Nhat Nguyen +Date: Thu Sep 20 19:53:30 2018 -0400 + + Propagate max_auto_id_timestamp in peer recovery (#33693) + + Today we don't store the auto-generated timestamp of append-only + operations in Lucene; and assign -1 to every index operations + constructed from LuceneChangesSnapshot. This looks innocent but it + generates duplicate documents on a replica if a retry append-only + arrives first via peer-recovery; then an original append-only arrives + via replication. Since the retry append-only (delivered via recovery) + does not have timestamp, the replica will happily optimizes the original + request while it should not. + + This change transmits the max auto-generated timestamp from the primary + to replicas before translog phase in peer recovery. This timestamp will + prevent replicas from optimizing append-only requests if retry + counterparts have been processed. + + Relates #33656 + Relates #33222 + +commit 6646bcb06581042f66bf72722954330145074b19 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Thu Sep 20 22:14:56 2018 +0200 + + [DOCS] Fix list formatting in TESTING.asciidoc (#33889) + + Fix the steps listed in `Testing packaging` to + ``` + 1. step1 + 2. step2 + -------------------------------------- + vagrant plugin install vagrant-cachier + -------------------------------------- + 3. step3 + ``` + instead of + ``` + instead of: + 1. step1 + 2. step2 + -------------------------------------- + vagrant plugin install vagrant-cachier + -------------------------------------- + 1. step3 + ``` + +commit 822b067a3ec1de5dcf99b05012d42509c65e36d8 +Author: Jan Jíša +Date: Thu Sep 20 22:13:18 2018 +0200 + + Docs: Corrected typo in how to (#33910) + + max_context_length -> max_content_length + +commit dbe640535491fd0ba8e0ce0a45649f4ed9114035 +Author: Vladimir Dolzhenko +Date: Thu Sep 20 21:30:40 2018 +0200 + + mute RemoveCorruptedShardDataCommandTests.testCorruptedIndex + +commit 187f787f52c9e71a7f8a4167c0c34851e4baa91d +Author: David Turner +Date: Thu Sep 20 20:05:55 2018 +0100 + + [Zen2] Introduce LeaderChecker (#33024) + + It is important that follower nodes periodically check that their leader is + still healthy and that they remain part of its cluster. If these checks fail + repeatedly then followers should attempt to find and join a new leader, + possibly electing one in the process. The LeaderChecker, introduced in this + commit, performs these periodic checks and deals with retries. + +commit 76a1a863e3a86fc5472bc2ef29091944bd1ff836 +Author: Nhat Nguyen +Date: Thu Sep 20 13:44:26 2018 -0400 + + TEST: stop assertSeqNos if shards movement (#33875) + + Currently, assertSeqNos assumes that the cluster is stable at the end of + the test (i.e., no more shard movement). However, this assumption does + not always hold. In these cases, we can stop the assertion instead of + failing a test. + + Closes #33704 + +commit 28b1d41007d6f611e1774a2afe785710315bd05b +Author: Christoph Büscher +Date: Thu Sep 20 19:42:15 2018 +0200 + + Fix unused import checktyle issue + +commit 002f763c482d43867a3ffb381a223864c0acc98e +Author: Nhat Nguyen +Date: Thu Sep 20 13:21:11 2018 -0400 + + Restore local history from translog on promotion (#33616) + + If a shard was serving as a replica when another shard was promoted to + primary, then its Lucene index was reset to the global checkpoint. + However, if the new primary fails before the primary/replica resync + completes and we are now being promoted, we have to restore the reverted + operations by replaying the translog to avoid losing acknowledged writes. + + Relates #33473 + Relates #32867 + +commit b13a434f59ebabacc5932e20a09c5ea7fd398831 +Author: Nhat Nguyen +Date: Thu Sep 20 13:09:27 2018 -0400 + + Remove wrong assert in LocalCheckpointTrackerTests + + It's possible for the set "seqNos" to contain only the "unFinishedSeq" + in the testConcurrentReplica test. If this is the case, the call + `randomValueOtherThan` won't make any progress because the predicate + will never be false. + + This commit removes this expectation because it's incorrect and it's no + longer needed as we have a dedicated test to verify the contains method. + + Relates #33871 + +commit 77145bb4771876b5f068e530ad52ded9b8e11331 +Author: Christoph Büscher +Date: Thu Sep 20 18:56:12 2018 +0200 + + HLRC: Add support for reindex rethrottling (#33832) + + This change adds support for rethrottling reindex requests to the + RestHighLevelClient. + +commit b33c18d316ef641091c9a588d11ee9c0cd7ad7b6 +Author: Alan Woodward +Date: Thu Sep 20 17:31:41 2018 +0100 + + Move SoraniNormalizationFilterFactory to the common analysis plugin (#33892) + + Follow up to #25715 + +commit db327818dd064b009f9bbb17384174da742b9e76 +Author: Yannick Welsch +Date: Thu Sep 20 18:16:20 2018 +0200 + + [TEST] Enable DEBUG logging on testCreateShrinkIndexToN + +commit b9e2b5cfec81c6bfa652368926d0d4102521dd72 +Author: Nick Peihl +Date: Thu Sep 20 07:55:49 2018 -0700 + + Update geolite2 database in ingest geoip plugin (#33840) + + * Update geolite2 database in ingest geoip plugin + + * Fix integration tests + +commit 8e3a0fad9d71b9bf117975b1194a6019c1840d27 +Author: Dimitris Athanasiou +Date: Thu Sep 20 15:48:42 2018 +0100 + + [ML] Refactor job deletion logic into the transport action (#33891) + + The job deletion logic was scattered around a few places: + the transport action, the job manager and the deletion task. + Overloading the task with deletion logic also meant extra + dependencies in the core package which should be unnecessary. + + This commit consolidates all this logic into the transport action + and replaces the deletion task with a plain one that needs not be + aware of deletion logic. + +commit f963c298764142404d1e8c3d34f304152152b572 +Author: Nik Everett +Date: Thu Sep 20 10:42:48 2018 -0400 + + Logging: Drop Settings from some logger lookups (#33859) + + Drops `Settings` from some of the methods to lookup loggers and + deprecates another logger lookup that takes `Settings` because + `Settings` is no longer required to build a logger. + +commit 0b4a6ae97ce1e5b7b70863ac5d5dd580e482cc12 +Merge: 6551b4f651a 3522b9084b6 +Author: David Turner +Date: Thu Sep 20 15:17:47 2018 +0100 + + Merge commit '3522b9084b611c89ec4f06c1863542883840ed0e' into zen2 + +commit e37e5dfc04c0608a2ef45692302e5ab4c6e3c43b +Author: Jake Landis +Date: Thu Sep 20 08:33:07 2018 -0500 + + ingest: support simulate with verbose for pipeline processor (#33839) + + * ingest: support simulate with verbose for pipeline processor + + This change better supports the use of simulate?verbose with the + pipeline processor. Prior to this change any pipeline processors + executed with simulate?verbose would not show all intermediate + processors for the inner pipelines. + + This changes also moves the PipelineProcess and TrackingResultProcessor + classes to enable instance checks and to avoid overly public classes. + As well this updates the error message for when cycles are detected + in pipelines calling other pipelines. + +commit 019aeadb7d297d04df9dd7dc78367a72628ce293 +Author: Christoph Büscher +Date: Thu Sep 20 15:01:36 2018 +0200 + + HLRC: Reindex should support `requests_per_seconds` parameter (#33808) + + The high level Rest clients reindex method currently doesn't pass on the + "requests_per_second" that are optionally set in ReindexRequest through the Rest + layer. This change makes sure the value is added to the request parameters if + set and also includes it for the update-by-query and delete-by-query cases. + +commit 6fb7e49b22f792623845c419e78504577d88633b +Author: Andrei Stefan +Date: Thu Sep 20 15:29:53 2018 +0300 + + SQL: TRUNCATE and ROUND functions (#33779) + + * Added TRUNCATE function, modified ROUND to accept two parameters instead of one. Made the second parameter optional for both functions. + * Added documentation for both functions. + +commit 168a0b23fea796c74fdff4155ab9426ecabe44ed +Author: Marios Trivyzas +Date: Thu Sep 20 14:26:07 2018 +0200 + + SQL: Fix ANTL4 Grammar ambiguities. (#33854) + + Removed rules in the grammar that were superfluous, as they + are already "caught" other rules in the same context. + + Also switched to exact ambig detection for debug mode + + Fixes: #31885 + +commit 3522b9084b611c89ec4f06c1863542883840ed0e +Author: Simon Willnauer +Date: Thu Sep 20 13:43:11 2018 +0200 + + Introduce a `search_throttled` threadpool (#33732) + + Today all searches happen on the search threadpool which is the correct + behavior in almost any case. Yet, there are exceptions where for instance + searches searches should be passed through a single-thread + thread-pool to reduce impact on a node. This change adds a index-private setting that allows to mark an index as throttled for searches and forks off all non-stats searcher access to this thread-pool for indices that are marked as `index.search.throttled` + +commit f5a2ffc3f6bb279236ac900668354a592638dfd5 +Author: David Roberts +Date: Thu Sep 20 12:17:09 2018 +0100 + + [DOCS][ML] Document the ML find_file_structure endpoint (#33723) + + Relates #33471 + Relates #33630 + +commit c041e943494dbf42081f4c56e306263688bae569 +Author: David Turner +Date: Thu Sep 20 11:17:19 2018 +0100 + + Test that transient settings beat persistent ones (#33818) + + Transient settings override persistent settings, but in fact all of the tests + that run as part of `:server:test` and `:server:integTest` will pass if the + precedence is changed to be the other way round. This change adds a test that + verifies the precedence is as documented. + +commit 6d3a7910f82e5c7be04a155a325723f829f26db3 +Author: David Turner +Date: Thu Sep 20 10:45:28 2018 +0100 + + Remove no-op test (#33861) + + This test was introduced when muting some other tests in #32498, but not + removed when the tests were unmuted in #32514. + +commit 8d50c102081903d3faf3fa0d1204539677a6be24 +Author: Tim Vernum +Date: Thu Sep 20 18:20:13 2018 +1000 + + Mute ShrinkIndexIT.testCreateShrinkIndexToN on Windows + + Relates: #33857 + +commit b1cc58e425b4d960feede256650b542bff04964d +Author: Daniel Mitterdorfer +Date: Thu Sep 20 08:59:53 2018 +0200 + + Allow to clear the fielddata cache per field + + With this commit we clear the fielddata cache per field as it is + supposed to be. Previously we retrieved the proper field from the cache + but then cleared the entire cache anyway. + + Closes #33798 + Relates #33807 + +commit 384ce58535fcc56dd95c99d0727e0844d98b8f53 +Author: Martijn van Groningen +Date: Thu Sep 20 08:55:57 2018 +0200 + + removed unused fields + +commit 3ca53437a27bf42502e3456a01a37b0ec5b695c1 +Author: Tim Vernum +Date: Thu Sep 20 16:24:02 2018 +1000 + + Mute SimpleSecurityNetty4TransportTests.testRenegotiation + + This test has failed 3 times in the last 3 days + + Relates: #33772 + +commit 44c7c4b166977044b9a1f35f5b881be3821a9650 +Author: Martijn van Groningen +Date: Thu Sep 20 07:16:20 2018 +0200 + + [CCR] Add auto follow stats api (#33801) + + GET /_ccr/auto_follow/stats + + Returns: + + ``` + { + "number_of_successful_follow_indices": ... + "number_of_failed_follow_indices": ... + "number_of_failed_remote_cluster_state_requests": ... + "recent_auto_follow_errors": [ + ... + ] + } + ``` + + Relates to #33007 + +commit 1f1ebb4656098b2f50f68d7cfae21c456c68259b +Author: Tim Vernum +Date: Thu Sep 20 13:17:29 2018 +1000 + + Add additional null check in _cat/shards + + The target of the func lambda may be null (e.g. in a mixed cluster + where older nodes lack some of the values) + + Relates: #33858 / 331caba + Closes #33877 + +commit ff934e3dcd5e4000a3f9946996d01c066ab39134 +Author: Tim Vernum +Date: Thu Sep 20 14:06:40 2018 +1000 + + Mute broken test on MacOS + + Seems to be triggered by 0cf0d73 + See: https://github.com/elastic/elasticsearch/issues/33879 + +commit 05bf9dc2e816840fe2b9b8505dfbc65d98ad7cc0 +Author: Nhat Nguyen +Date: Wed Sep 19 20:29:36 2018 -0400 + + Add contains method to LocalCheckpointTracker (#33871) + + This change adds "contains" method to LocalCheckpointTracker. + One of the use cases is to check if a given operation has been processed + in an engine or not by looking up its seq_no in LocalCheckpointTracker. + + Relates #33656 + +commit 90de436e55f65b6da6bc31898df053d73783f6d6 +Author: Gordon Brown +Date: Wed Sep 19 14:50:48 2018 -0600 + + Use custom index metadata for ILM state (#33783) + + Using index settings for ILM state is fragile and exposes too much + information that doesn't need to be exposed. Using custom index metadata + is more resilient and allows more controlled access to internal + information. + + As part of these changes, moves away from using defaults for ILM-related + values, in favor of using null values to clearly indicate that the value is not + present. + +commit 4767a016a5acae729c35be1954a27f5209911cca +Author: Benjamin Trent +Date: Wed Sep 19 13:35:09 2018 -0700 + + Adding node_count to ML Usage (#33850) (#33863) + +commit 839a677557d1a3a3218eda60fdc7315c49e1a2e0 +Author: Jason Tedor +Date: Wed Sep 19 16:18:54 2018 -0400 + + Do not override named S3 client credentials (#33793) + + In cases when mixed secure S3 client credentials and insecure S3 client + credentials were used (that is, those defined on the repository), we + were overriding the credentials from the repository using insecure + settings to all the repositories. This commit fixes this by not mixing + up repositories that use insecure settings with those that use secure + settings. + +commit 26c4f1fb6c1209bf5280fa353cf120ad825c7c04 +Author: Nik Everett +Date: Wed Sep 19 15:21:29 2018 -0400 + + Core: Default node.name to the hostname (#33677) + + Changes the default of the `node.name` setting to the hostname of the + machine on which Elasticsearch is running. Previously it was the first 8 + characters of the node id. This had the advantage of producing a unique + name even when the node name isn't configured but the disadvantage of + being unrecognizable and not being available until fairly late in the + startup process. Of particular interest is that it isn't available until + after logging is configured. This forces us to use a volatile read + whenever we add the node name to the log. + + Using the hostname is available immediately on startup and is generally + recognizable but has the disadvantage of not being unique when run on + machines that don't set their hostname or when multiple elasticsearch + processes are run on the same host. I believe that, taken together, it + is better to default to the hostname. + + 1. Running multiple copies of Elasticsearch on the same node is a fairly + advanced feature. We do it all the as part of the elasticsearch build + for testing but we make sure to set the node name then. + 2. That the node.name defaults to some flavor of "localhost" on an + unconfigured box feels like it isn't going to come up too much in + production. I expect most production deployments to at least set the + hostname. + + As a bonus, production deployments need no longer set the node name in + most cases. At least in my experience most folks set it to the hostname + anyway. + +commit 3df285d9f0be822165661e7cc0c3781bf5fe0d8a +Author: Jack Conradson +Date: Wed Sep 19 12:19:44 2018 -0700 + + Painless: Rename *Binding classes to *ClassBinding. (#33865) + + With the upcoming instance bindings, the singular *Binding name isn't descriptive enough + with multiple binding types. This renames the existing *Binding classes to *ClassBinding. + Mechanical change (with some error messages changed from binding to class binding by + hand). + +commit c764012347edce75ec2d1f8951f07692e923b218 +Author: Julie Tibshirani +Date: Wed Sep 19 12:17:08 2018 -0700 + + Create a WatchStatus class for the high-level REST client. (#33527) + + This class will be used in a few of the watcher responses ('get watch', 'ack watch', etc.), so it's being introduced first in its own PR. + +commit a92dda2e7e2ee7c0d6a5c12d5b50cb8b3d9bced3 +Author: Simon Willnauer +Date: Wed Sep 19 20:35:57 2018 +0200 + + Move CompletionStats into the Engine (#33847) + + By moving CompletionStats into the engine we can easily cache the stats for + read-only engines if necessary. It also moves the responsibiltiy out of IndexShard + which has quiet some complexity already. + + Relates to #33835 + +commit 0fa5758bc672914b61176df472fd2c101aa5abd9 +Author: Simon Willnauer +Date: Wed Sep 19 20:34:54 2018 +0200 + + Fix potential NPE in `_cat/shards/` with partial CommonStats (#33858) + + Today if we fetch common stats from a shard we might get a partial response + if the shard is closed while we fetch the stats. This causes hard to track and + reproduce NPEs. This change streamlines null checking to ensure we only render + stats we actually received. + +commit 3ede13a4546b30644f51fc14f7f917bc925d5a37 +Author: Nik Everett +Date: Wed Sep 19 14:34:02 2018 -0400 + + Test framework fall cleaning (#33423) + + Wraps all lines in our test framework at 140 characters because that is + our standard line length and removes all of the checkstyle suppressions + for the test framework. + + Drops most of `ModuleTestCase` because it isn't used and we're moving + away from using guice in the way that it wants to test anyway. Also + switches a few classes that extend it but don't use it to extend + `ESTestCase` instead. + +commit 81e9150c7a60ca6efac26c57cd5aaa60b04641a4 +Merge: 27dd25857bc 6ec12bef0d6 +Author: Lee Hinman +Date: Wed Sep 19 09:43:26 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 6ec12bef0d6258996d5d4d7b56b2c3c9a9805c74 +Author: Simon Willnauer +Date: Wed Sep 19 17:07:13 2018 +0200 + + Add missing IndexShard#readAllowed() + + This was lost in #33835 + +commit 5107949402855d72125d2dbec5a4aaf025201d0f +Author: Alan Woodward +Date: Wed Sep 19 15:52:14 2018 +0100 + + Allow TokenFilterFactories to rewrite themselves against their preceding chain (#33702) + + We currently special-case SynonymFilterFactory and SynonymGraphFilterFactory, which need to + know their predecessors in the analysis chain in order to correctly analyze their synonym lists. This + special-casing doesn't work with Referring filter factories, such as the Multiplexer or Conditional + filters. We also have a number of filters (eg the Multiplexer) that will break synonyms when they + appear before them in a chain, because they produce multiple tokens at the same position. + + This commit adds two methods to the TokenFilterFactory interface. + + * `getChainAwareTokenFilterFactory()` allows a filter factory to rewrite itself against its preceding + filter chain, or to resolve references to other filters. It replaces `ReferringFilterFactory` and + `CustomAnalyzerProvider.checkAndApplySynonymFilter`, and by default returns `this`. + * `getSynonymFilter()` defines whether or not a filter should be applied when building a synonym + list `Analyzer`. By default it returns `true`. + + Fixes #33609 + +commit 4190a9f1e909d8c9a42a3e8fe44c1cbdd9000e7e +Author: Benjamin Trent +Date: Wed Sep 19 07:42:26 2018 -0700 + + Delete custom index if the only contained job is deleted (#33788) + + * Delete custom index if the only contained job is deleted + +commit c118581617dd209e894c7d188fcba334e4c25766 +Author: markharwood +Date: Wed Sep 19 15:16:14 2018 +0100 + + Test fix - Graph connections could appear in different orders + Graph connections could appear in different orders based on insertion sequence + + Closes #33686 + +commit 546e7361edde23039d7b82f51fcfc5ccb34c86d6 +Author: Christoph Büscher +Date: Wed Sep 19 15:58:35 2018 +0200 + + [Tests] Nudge wait time in RemoteClusterServiceTests (#33853) + + This test occasionally fails in `testCollectSearchShards` waiting on what seems + to be a search request to a remote cluster for one second. Given that the test + fails here very rarely I suspect maybe one second is very rarely not enough so + we could fix it by increasing the max wait time slightly. + + Closes #33852 + +commit d78966c4c73df184484287ce303ad0c3976f8494 +Author: Shaunak Kashyap +Date: Wed Sep 19 04:49:54 2018 -0700 + + Fixing assertions in integration test (#33833) + +commit 6551b4f651a14b0477f8544da0fa35ca9969abfa +Author: Yannick Welsch +Date: Wed Sep 19 13:36:11 2018 +0200 + + Zen2: Integrate publication pipeline into Coordinator (#33771) + + Replaces the mock integration of Publication in CoordinatorTests by the real thing. + +commit d9947c631a9d398172b717b7864e8ffbcebcdb3c +Author: Martijn van Groningen +Date: Wed Sep 19 13:13:20 2018 +0200 + + [CCR] Rename idle_shard_retry_delay to poll_timout in auto follow patterns (#33821) + +commit 68c0a295787f0f0549b01738740372c0abe4e3a0 +Author: David Kyle +Date: Wed Sep 19 10:20:21 2018 +0100 + + HLRC: Delete ML calendar (#33775) + +commit 10009434bfc5a5352cf367a3a8450a6a0529c814 +Merge: 758b2f91112 0c77f45dc66 +Author: Yannick Welsch +Date: Wed Sep 19 11:18:01 2018 +0200 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 0c77f45dc66a448fc37e5cf8a1984282083fc28f +Author: Simon Willnauer +Date: Wed Sep 19 11:03:11 2018 +0200 + + Move DocsStats into Engine (#33835) + + By moving DocStats into the engine we can easily cache the stats for + read-only engines if necessary. It also moves the responsibility out of IndexShard + which has quiet some complexity already. + +commit 6f3b3338ba73f1ebc50836758b772d7c57b8852f +Author: Christoph Büscher +Date: Wed Sep 19 10:34:53 2018 +0200 + + [Docs] Clarify accessing Date methods in painless (#33560) + + The documentation currently tells users to use `doc['event_date'].value.getMillis` to access + milliseconds in a date. It turns out the way it works is `doc['event_date'].value.millis`. This + change corrects this and gives a hint at how other date related methods work. + +commit a3e8b831ee3c83ccbecc28734d40a55d5cbddfec +Author: Vladimir Dolzhenko +Date: Wed Sep 19 10:28:22 2018 +0200 + + add elasticsearch-shard tool (#32281) + + Relates #31389 + +commit 251489d59a3087819ff3918df316ac145c5889b2 +Author: Simon Willnauer +Date: Wed Sep 19 10:18:03 2018 +0200 + + Cut over to unwrap segment reader (#33843) + + The fix in #33757 introduces some workaround since FilterCodecReader didn't + support unwrapping. This cuts over to a more elegant fix to access the readers + segment infos. + +commit d22b383b9cb5f17e7f2dbdd6524b3b80f56f2cdc +Author: Marios Trivyzas +Date: Wed Sep 19 10:16:24 2018 +0200 + + SQL: Fix issue with options for QUERY() and MATCH(). (#33828) + + Previously multiple comma separated lists of options where not + recognized correctly which resulted in only the last of them + to be taked into account, e.g.: + + For the following query: + SELECT * FROM test WHERE QUERY('search', 'default_field=foo', 'default_operator=and')" + only the `default_operator=and` was finally passed to the ES query. + + Fixes: #32602 + +commit c9765d5fb91ef549d51eaa71daf7ed12e461df77 +Author: David Turner +Date: Wed Sep 19 08:36:03 2018 +0100 + + Emphasize that filesystem-level backups don't work (#33102) + + It is not obvious that a filesystem-level backup may capture an inconsistent + set of files that may fail on restore, or (worse) succeed having silently + discarded some data. This change spells the out, and reorganises the first page + or so of the snapshot/restore docs to make this warning fit more nicely. + +commit 61e1df0274cb74d3c90a7ca4504d233114ae4e85 +Author: Jim Ferenczi +Date: Wed Sep 19 09:28:38 2018 +0200 + + Use the global doc id to generate a random score (#33599) + + This commit changes the random_score function to use the global docID of the document + rather than the segment docID to generate random scores. As a result documents that have + the same segment docID within the shard will generate different scores. + +commit c4261bab44742724b2e04c43a4abef3b7cba08c0 +Author: Adrien Grand +Date: Wed Sep 19 09:19:13 2018 +0200 + + Add minimal sanity checks to custom/scripted similarities. (#33564) + + Add minimal sanity checks to custom/scripted similarities. + + Lucene 8 introduced more constraints on similarities, in particular: + - scores must not be negative, + - scores must not decrease when term freq increases, + - scores must not increase when norm (interpreted as an unsigned long) + increases. + + We can't check every single case, but could at least run some sanity checks. + + Relates #33309 + +commit 7f473b683d4f7426d3aa13cb618b6330a09ba309 +Author: Ignacio Vera +Date: Tue Sep 18 23:32:16 2018 -0700 + + Profiler: Don’t profile NEXTDOC for ConstantScoreQuery. (#33196) + + * Profiler: Don’t profile NEXTDOC for ConstantScoreQuery. + + A ConstantScore query will return the iterator of its inner query. + However, when profiling, the constant score query is wrapped separately + from its inner query, which distorts the times emitted by the profiler. + Return the iterator directly in such a case. + + Closes #23430 + +commit 013b64a07c0f1580352f6188b0dc300ce97c5057 +Author: Martijn van Groningen +Date: Wed Sep 19 07:18:24 2018 +0200 + + [CCR] Change FollowIndexAction.Request class to be more user friendly (#33810) + + Instead of having one constructor that accepts all arguments, all parameters + should be provided via setters. Only leader and follower index are required + arguments. This makes using this class in tests and transport client easier. + +commit c6e3231ef38dda152c9fcc2b74519a9a1e33e0e9 +Author: Andrei Stefan +Date: Wed Sep 19 07:33:15 2018 +0300 + + SQL: day and month name functions tests locale providers enforcement (#33653) + + DAYNAME and MONTHNAME functions tests will be skipped if the right JVM parameter (-Djava.locale.providers=COMPAT) is not used in unit testing environment + +commit 0cf0d73813e7732234e2a89611f05b9faa165ad2 +Author: Armin Braun +Date: Wed Sep 19 06:05:36 2018 +0200 + + TESTS: Set SO_LINGER = 0 for MockNioTransport (#32560) + + * TESTS: Set SO_LINGER = 0 for MockNioTransport + + * Prevents lingering sockets in TIME_WAIT piling up during test runs and leading to port collisions that manifest as timeouts + * Fixes #32552 + +commit 27dd25857bca3b4cadef5d04994b0e9f71784cdb +Author: Lee Hinman +Date: Tue Sep 18 17:07:57 2018 -0600 + + Rebuild step on PolicyStepsRegistry.getStep (#33780) + + This moves away from caching a list of steps for a current phase, instead + rebuilding the necessary step from the phase JSON stored in the index's + metadata. + + Relates to #29823 + +commit 99513b306e84b22b83cefde4c07ed12f576ffc09 +Author: Ryan Ernst +Date: Tue Sep 18 13:11:46 2018 -0700 + + Test: Relax jarhell gradle test (#33787) + + Gradle can sometimes emit mixed log lines due to how we spawn things in + separate processes. This commit changes the jarhell integ test to only + look for the exception and message, instead of including the outer part + about the exception in thread main. + + closes #33774 + +commit 11a55d230709bd52c916df108310a618d127140d +Author: Lee Hinman +Date: Tue Sep 18 14:02:09 2018 -0600 + + [TEST] Handle an IndexLifecycleService that has not started up + +commit c87cff22b4b41925053cf05e0f0dd171959e3d7e +Merge: 94a66c556d8 805a12361fe +Author: Lee Hinman +Date: Tue Sep 18 13:57:41 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 805a12361fe30f940121627a8a9398e7d93cffef +Author: Martijn van Groningen +Date: Tue Sep 18 21:47:02 2018 +0200 + + [CCR] Fail with a descriptive error if leader index does not exist (#33797) + + Closes #33737 + +commit f4cbbcf98b8b0acfdabbc2a8dfee28bd3536bbca +Author: Zachary Tong +Date: Tue Sep 18 15:25:20 2018 -0400 + + Add ES version 6.4.2 (#33831) + + Version and properties files + +commit c6462057a15289bfb69a97441159b555b37d7d80 +Author: Armin Braun +Date: Tue Sep 18 20:43:31 2018 +0200 + + MINOR: Remove Some Dead Code in Scripting (#33800) + + * The is default check method is not used in ScriptType + * The removed vars on ExpressionSearchScript are unused + +commit 9026c3ee92f1f13630a989023acd8c47829e3ec0 +Author: Simon Willnauer +Date: Tue Sep 18 19:53:42 2018 +0200 + + Ensure realtime `_get` and `_termvectors` don't run on the network thread (#33814) + + The change in #27500 introduces this regression that causes `_get` and `_term_vector` + actions to run on the network thread if the realtime flag is set. + This fixes the issue by delegating to the super method forking on the corresponding threadpool. + +commit 98ccd9496275591fb81b5a24e98f5bd7ac3dc442 +Author: Simon Willnauer +Date: Tue Sep 18 19:53:26 2018 +0200 + + Factor out a ChannelActionListener (#33819) + + We use similar / same concepts in SerachTransportService and HandledTransportAction but both + duplicate the efforts with slightly different implementation details. This streamlines + sending responses / exceptions back to a channel in an ActionListener with appropriate logging. + +commit 241c74efb26b3d4a774bc9e8b8450d029c91a83a +Author: Jim Ferenczi +Date: Tue Sep 18 18:16:40 2018 +0200 + + upgrade to a new snapshot of Lucene 8 (7d0a7782fa) (#33812) + +commit bc12a948b5c7914f479d0de4cf3d4e3fa7a3fcf2 +Author: Nik Everett +Date: Tue Sep 18 12:04:35 2018 -0400 + + Checkstyle: Package declarations (#33784) + + Make sure that all java files have a package declaration and that all of + the package declarations line up with the directory structure. This + would have caught the bug that I caused in + 190ea9a6def9082348d983b16420ef02607d4c17 and fixed in + b6d68bd805f1858a0210e381402236dea1d42509. + +commit 91e45ca21b9490fe46d39aa6240a289a59304ab8 +Author: Costin Leau +Date: Tue Sep 18 18:51:48 2018 +0300 + + SQL: Better handling of number parsing exceptions (#33776) + + Add proper exceptions in case the parsing of numbers (too large, invalid + format) fails. + + Close #33622 + +commit 7bed91549997a5f5736eae4bae1eb7f3f38843d7 +Author: Tomas Della Vedova +Date: Tue Sep 18 17:05:10 2018 +0200 + + [DOCS] Fixed list formatting (#32963) + +commit 3928921a1d38ccff2bd215f2cccd5876b50503b8 +Author: Tim Heckel +Date: Tue Sep 18 09:59:26 2018 -0500 + + [DOCS] Update scroll.asciidoc (#32530) + +commit 32ee6148d2eb1865e491e162ad5bd7e595cf0bb8 +Author: Abdon Pijpelink +Date: Tue Sep 18 16:57:33 2018 +0200 + + [DOCS] Clarify scoring for multi_match phrase type (#32672) + + The original statement "Runs a match_phrase query on each field and combines the _score from each field." for the phrase type is a but misleading. The phrase type behaves like the best_fields type and does not combine the scores of each fields. + +commit 3596512e6a3eeb74316f035d0dea1b31eafb60f3 +Author: Dan Tennery-Spalding +Date: Tue Sep 18 07:46:22 2018 -0700 + + [DOCS] Corrected several grammar errors (#33781) + +commit 9fe5a273aac70d0685300de1d012566fbdada4f9 +Author: Martijn van Groningen +Date: Tue Sep 18 15:55:16 2018 +0200 + + [TEST] handle failed search requests differently + +commit 8e0d74adadab17b8aff4dfdf9e5a37fcaffa145f +Author: Dimitris Athanasiou +Date: Tue Sep 18 12:56:37 2018 +0100 + + [ML][HLRC] Remove deleted property from Job (#33763) + + The deleted property is meant to be used internally. + Users of the client should not need interact with that property. + +commit 0d4683850c069d973050880f63970c7fa57b72af +Author: Andrei Stefan +Date: Tue Sep 18 14:12:18 2018 +0300 + + Moved the problematic tests to the tests file that is not considered when certain locales are used (#33785) + +commit 421f58e17243a08144a1f819d2896da4cef5877b +Author: David Turner +Date: Tue Sep 18 12:01:16 2018 +0100 + + Remove discovery-file plugin (#33257) + + In #33241 we moved the file-based discovery functionality to core + Elasticsearch, but preserved the `discovery-file` plugin, and support for the + existing location of the `unicast_hosts.txt` file, for BWC reasons. This commit + completes the removal of this plugin. + +commit 9543992d8e1c15215055942a15b670d7b0213d75 +Author: David Kyle +Date: Tue Sep 18 11:51:11 2018 +0100 + + HLRC: Get ML calendars (#33760) + +commit 758b2f91112b865d9e77d6b086b05a6d9b62626e +Author: Yannick Welsch +Date: Tue Sep 18 11:48:24 2018 +0200 + + Zen2: Add DisruptableMockTransport (#33713) + + Adds a mock transport implementation that allows to simulate network disruptions. + +commit 2fa09f062e5fe4165b457fb2b42ff4b65acdb8c7 +Author: markharwood +Date: Tue Sep 18 10:25:27 2018 +0100 + + New plugin - Annotated_text field type (#30364) + + New plugin for annotated_text field type. + Largely a copy of `text` field type but adds ability to include markdown-like syntax in the text. + The “AnnotatedText” class parses text+markup and converts into plain text and AnnotationTokens. + The annotation token values are injected unchanged alongside the regular text tokens to provide a + form of additional indexed overlay useful in positional searches and highlighting. + Annotated_text fields do not support fielddata as we want to phase this out. + Also includes a new "annotated" highlighter type that retains annotations and merges in search + hits as additional annotation markup. + + Closes #29467 + +commit ab9c28a2b11142f86d6ca2c3bb52ab6bd5d39558 +Author: Costin Leau +Date: Tue Sep 18 11:44:52 2018 +0300 + + SQL: Grammar tweak for number declarations (#33767) + + Consider plus and minus as part of a number declaration (to avoid the + minus be treated as a negation). + + Close #33765 + +commit 87cedef3cfca9b8a03100f7de4a3c1d40ac260d9 +Author: Armin Braun +Date: Tue Sep 18 10:29:02 2018 +0200 + + NETWORKING:Def CName in Http Publish Addr to True (#33631) + + * Follow up to #32806 setting the setting to true for 7.x + +commit 139128856a5f63deb41243756e2b3b3a843fa57c +Author: Alexander Reelsen +Date: Tue Sep 18 10:25:16 2018 +0200 + + Watcher: Use Bulkprocessor in HistoryStore/TriggeredWatchStore (#32490) + + Currently a watch execution results in one bulk request, when the + triggered watches are written into the that index, that need to be + executed. However the update of the watch status, the creation of the + watch history entry as well as the deletion of the triggered watches + index are all single document operations. + + This can have quite a negative impact, once you are executing a lot of + watches, as each execution results in 4 documents writes, three of them + being single document actions. + + This commit switches to a bulk processor instead of a single document + action for writing watch history entries and deleting triggered watch + entries. However the defaults are to run synchronous as before because + the number of concurrent requests is set to 0. This also fixes a bug, + where the deletion of the triggered watch entry was done asynchronously. + + However if you have a high number of watches being executed, you can + configure watcher to delete the triggered watches entries as well as + writing the watch history entries via bulk requests. + + The triggered watches deletions should still happen in a timely manner, + where as the history entries might actually be bound by size as one + entry can easily have 20kb. + + The following settings have been added: + + - xpack.watcher.bulk.actions (default 1) + - xpack.watcher.bulk.concurrent_requests (default 0) + - xpack.watcher.bulk.flush_interval (default 1s) + - xpack.watcher.bulk.size (default 1mb) + + The drawback of this is of course, that on a node outage you might end + up with watch history entries not being written or watches needing to be + executing again because they have not been deleted from the triggered + watches index. The window of these two cases increases configuring the bulk processor to wait to reach certain thresholds. + +commit e075b872f620c4ac2abda18d281db332c7e69847 +Author: Alexander Reelsen +Date: Tue Sep 18 10:14:12 2018 +0200 + + Dependencies: Update javax.mail in watcher to 1.6.2 (#33664) + +commit 47b86d6e6ae615d5284b3b9bdc931c41dd3befb5 +Author: Martijn van Groningen +Date: Tue Sep 18 09:43:50 2018 +0200 + + [CCR] Changed AutoFollowCoordinator to keep track of certain statistics (#33684) + + The following stats are being kept track of: + 1) The total number of times that auto following a leader index succeed. + 2) The total number of times that auto following a leader index failed. + 3) The total number of times that fetching a remote cluster state failed. + 4) The most recent 256 auto follow failures per auto leader index + (e.g. create_and_follow api call fails) or cluster alias + (e.g. fetching remote cluster state fails). + + Each auto follow run now produces a result that is being used to update + the stats being kept track of in AutoFollowCoordinator. + + Relates to #33007 + +commit 615f494c7783bf2e689392213954a341812b0802 +Author: Armin Braun +Date: Tue Sep 18 07:25:22 2018 +0200 + + MINOR: Drop Redundant Ctx. Check in ScriptService (#33782) + + * MINOR: Drop Redundant Ctx. Check in ScriptService + + * This check is completely redundant, the expression script + engine will throw anyway (and with a similar message) for + those contexts that it cannot compile. Moreover, the update context + is not the only context that is not suported by the expression engine + at this point so handling the update context separately here makes + no sense. + +commit 2aba52de8f9315b0e384e1c657d7b0401d26a1b0 +Author: Shaunak Kashyap +Date: Mon Sep 17 18:33:43 2018 -0700 + + Implement xpack.monitoring.elasticsearch.collection.enabled setting (#33474) + + * Implement xpack.monitoring.elasticsearch.collection.enabled setting + + * Fixing line lengths + + * Updating constructor calls in test + + * Removing unused import + + * Fixing line lengths in test classes + + * Make monitoringService.isElasticsearchCollectionEnabled() return true for tests + + * Remove wrong expectation + + * Adding unit tests for new flag to be false + + * Fixing line wrapping/indentation for better readability + + * Adding docs + + * Fixing logic in ClusterStatsCollector::shouldCollect + + * Rebasing with master and resolving conflicts + + * Simplifying implementation by gating scheduling + + * Doc fixes / improvements + + * Making methods package private + + * Fixing wording + + * Fixing method access + +commit a95226bdaeaf13347d1bbd2761e3eeaead47c145 +Author: Shaunak Kashyap +Date: Mon Sep 17 18:29:30 2018 -0700 + + [Monitoring] Removing unused version.* fields (#33584) + + This PR removes fields that are not actually used by the Monitoring UI. This will greatly simplify the eventual migration to using Metricbeat for monitoring Elasticsearch (see https://github.com/elastic/beats/pull/8260#discussion_r215885868 for more context and discussion around removing these fields from ES collection). + +commit 012b9c7539471ecde25515cf49b12846af0bc9e0 +Author: ben5556 <39107453+ben5556@users.noreply.github.com> +Date: Tue Sep 18 13:21:15 2018 +1200 + + Corrected aggregation name to match the example (#33786) + +commit 94a66c556d82f4490b5ac7a05ab970342ab98d81 +Author: Tal Levy +Date: Mon Sep 17 17:00:00 2018 -0700 + + add phase execution info to ILM Explain API (#33488) + + adds a section for phase execution to the Explain API. + + This contains + + - phase definition + - policy name + - policy version + - modified date + +commit 42e106fb226b767788859a32be86878b2ae10653 +Author: Michael Basnight +Date: Mon Sep 17 17:38:30 2018 -0500 + + HLRC: split indices request converters (#33433) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the IndicesClient + request converters. + +commit e686909768b163a0999e98efe7adb3046eeef3df +Author: Ryan Ernst +Date: Mon Sep 17 15:10:16 2018 -0700 + + Build: Change test task sanity check to be per project (#33544) + + This commit changes the sanity check which ensures the test task was + properly replaced with randomized testing to have a per project check, + isntead of a global one. The previous global check assumed all test + tasks within the root project and below should be randomized testing, + but that is not the case for a multi project in which only one project + is an elasticsearch plugin. While the new check is not able to emit all + of the failed replacements in one error message, the efficacy of the + check remains. + +commit 15f30d689bba00b8662ed047b5650af1920bfce5 +Author: Martijn van Groningen +Date: Mon Sep 17 22:33:37 2018 +0200 + + [CCR] Do not unnecessarily wrap fetch exception in a ElasticSearch exception and (#33777) + + * [CCR] Do not unnecessarily wrap fetch exception in a ElasticSearch exception and + properly map fetch_exception.exception field as object. + + The extra caused by level is not necessary here: + + ``` + "fetch_exceptions": [ + { + "from_seq_no": 1, + "retries": 106, + "exception": { + "type": "exception", + "reason": "[index1] IndexNotFoundException[no such index]", + "caused_by": { + "type": "index_not_found_exception", + "reason": "no such index", + "index_uuid": "_na_", + "index": "index1" + } + } + } + ], + ``` + +commit a5bad4d92c564f7f5c843d12a4dcb11b8c757735 +Author: Or Bin +Date: Mon Sep 17 22:35:55 2018 +0300 + + Docs: Fixed a grammatical mistake: 'a HTTP ...' -> 'an HTTP ...' (#33744) + + Fixed a grammatical mistake: 'a HTTP ...' -> 'an HTTP ...' + + Closes #33728 + +commit d8dc0425147c134dc72de72fdf7fd69088c76f53 +Author: Martijn van Groningen +Date: Mon Sep 17 19:47:40 2018 +0200 + + [CCR] Handle leader index with no mapping correctly (#33770) + + When a leader index is created, it may not have a mapping yet. + Currently if you follow such an index the shard follow tasks fail with + NoSuchElementException, because they expect a single mapping. + + This commit fixes that, by allowing that a leader index does not yet have + a mapping. + +commit 7ff11b4ae1c84b8190b919d5ddb7d6e63efe5062 +Merge: e6cbaa5a78f 48a5b45d28a +Author: Lee Hinman +Date: Mon Sep 17 10:41:10 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 48a5b45d28a0740231128c00a04586b5a2c1cb12 +Author: Simon Willnauer +Date: Mon Sep 17 18:18:58 2018 +0200 + + Ensure fully deleted segments are accounted for correctly (#33757) + + We can't rely on the leaf reader ordinal in a wrapped reader since + it might not correspond to the ordinal in the SegmentInfos for it's + SegmentCommitInfo. + + Relates to #32844 + Closes #33689 + Closes #33755 + +commit 7046cc467f63186fc5b9e7d388f4a9ceb25b8bed +Author: Martijn van Groningen +Date: Mon Sep 17 18:08:19 2018 +0200 + + [CCR] Make index.xpack.ccr.following_index an internal setting (#33768) + +commit 5d2a01dcc3aa36e95d5a5b187bbfef755d53622b +Author: Martijn van Groningen +Date: Mon Sep 17 18:00:16 2018 +0200 + + [CCR] Fail with a good error if a follow index does not have ccr metadata (#33761) + + instead of a NPE. + +commit 4d0bea705cd10fcfbec3e70d38712ae35eba5b9d +Author: Vladimir Dolzhenko +Date: Mon Sep 17 17:56:47 2018 +0200 + + Do not report negative free bytes for DiskThresholdDecider#canAllocate (#33641) + + Do not report negative free bytes for DiskThresholdDecider#canAllocate (#33641) + + Closes #33596 + +commit 7f6c13037c95f151882fe20f064177ea76ce833e +Author: David Turner +Date: Mon Sep 17 16:23:55 2018 +0100 + + Mark testMonitoringService as @AwaitsFix + +commit a654f21599907266d61253c163354fe132326558 +Author: Armin Braun +Date: Mon Sep 17 16:38:44 2018 +0200 + + TESTS: Fix Concurent Remote Connection Updates (#33707) + + * Same fix idea as in #10666a4 to prevent background + threads trying to reconnect after the tests are done from + throwing `ExecutionCancelledException` and breaking the test + * Closes #30714 + +commit 2d81fc38732f51c480684af1084688c863f95368 +Author: Jason Tedor +Date: Mon Sep 17 09:59:22 2018 -0400 + + Keep CCR REST API specification with all of X-Pack (#33743) + + This commit moves the CCR REST API specification out of the CCR + sub-project to locate them with the rest of the REST API specifications + for X-Pack. + +commit c79fbea9235d09477fd5c95dffdbfcc28b5ebb60 +Author: David Turner +Date: Mon Sep 17 15:00:30 2018 +0200 + + [Zen2] Implement basic cluster formation (#33668) + + This PR integrates the following pieces of machinery in the Coordinator: + + - discovery + - pre-voting + - randomised election scheduling + - joining (of a new master) + - publication of cluster state updates + + Together, these things are everything needed to form a cluster. We therefore + also add the start of a test suite that allows us to assert higher-level + properties of the interactions between all these pieces of machinery, with as + little fake behaviour as possible. We assert one such property: "a cluster + successfully forms". + +commit b2413d2068f732353184e4fa378f8f20eb56886b +Author: Tanguy Leroux +Date: Mon Sep 17 13:37:39 2018 +0200 + + [Docs] Fix broken external links in HLRC Rollup documentation + + Another attempt. Introduced in #33521 + +commit 14d57c111524cd0504f058d07599b8c9e7359557 +Author: Bukhtawar +Date: Mon Sep 17 16:43:44 2018 +0530 + + Skip rebalancing when cluster_concurrent_rebalance threshold reached (#33329) + + Allows to skip shard balancing when the cluster_concurrent_rebalance threshold is already reached, which cuts down the time spent in the rebalance method of BalancedShardsAllocator. + +commit 5ca6f312058ec8c7fbac072c4648246e7dfb4957 +Author: Alpar Torok +Date: Mon Sep 17 14:09:28 2018 +0300 + + Move precommit task implementation to java (#33407) + + Replace precommit tasks that execute with Java implementations + +commit baa69a5ed50fc255c7656083741c72c9b990c3e4 +Author: Tanguy Leroux +Date: Mon Sep 17 13:02:11 2018 +0200 + + [Docs] Fix broken links in HLRC Rollup documentation + + Introduced in #33521 + +commit b06a082725ee8ab5e3755cb7990ca3bc94ae15a0 +Author: Adrien Grand +Date: Mon Sep 17 11:49:22 2018 +0200 + + Improve reproducibility of BigArraysTests. + + Close #33750 + +commit 5f9370f0ecbf319e5201a82e60fde59c6de003f0 +Author: Ioannis Kakavas +Date: Mon Sep 17 12:38:51 2018 +0300 + + [TESTS] Mute SSLDriverTests in JDK11 + + Relates: https://github.com/elastic/elasticsearch/issues/33751 + +commit 1f2a90cb3937dcc6a6164e24b3a5350d69283f91 +Author: Christoph Büscher +Date: Mon Sep 17 11:15:55 2018 +0200 + + Mute DateTimeUnitTests.testConversion + +commit 896b3864a87bfcb39c937cc2ee583767b1844870 +Author: Alpar Torok +Date: Mon Sep 17 12:04:57 2018 +0300 + + Favor running compile tasks before pre-commit (#33424) + +commit 6b0fc5382a41d87473a14c3aa846e10cb17ce27b +Author: David Turner +Date: Mon Sep 17 09:22:52 2018 +0100 + + Suppress DeadHostStateTests on Windows + +commit 01b3be917afd802cfe28dc0745c73f79abd91724 +Merge: 31e8781eaa3 e77835c6f5c +Author: Yannick Welsch +Date: Mon Sep 17 09:59:37 2018 +0200 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit e77835c6f5c143e0f8c4a8192528fb70b6c801df +Author: Tanguy Leroux +Date: Mon Sep 17 09:10:23 2018 +0200 + + Add create rollup job api to high level rest client (#33521) + + This commit adds the Create Rollup Job API to the high level REST + client. It supersedes #32703 and adds dedicated request/response + objects so that it does not depend on server side components. + + Related #29827 + +commit 34379887b4734f713db0e01b48835c5e7384d9ad +Author: Martijn van Groningen +Date: Mon Sep 17 07:51:34 2018 +0200 + + Make custom index metadata completely immutable (#33735) + + Currently `IndexMetadata#getCustomData(...)` wraps the custom metadata + in an unmodifiable map, but in case there is no entry for the specified + key then a NPE is thrown by Collections.unmodifiableMap(...). This is not + ideal in case callers like to throw an exception with a specific message. + (like in the case for ccr to indicate that the follow index was not created + by the create_and_follow api and therefor incompatible as follow index) + + I think making `DiffableStringMap` itself immutable is better then just wrapping + custom metadata with `Collections.unmodifiableMap(...)` in all methods that access it. + + Also removed the `equals()`, `hashcode()` and to `toString()` methods of + `DiffableStringMap`, because `AbstractMap` already implements these methods. + +commit 481f8a9a0766e82fb8f0d0bef30ce6e5ca91cd33 +Author: Martijn van Groningen +Date: Mon Sep 17 07:29:00 2018 +0200 + + [CCR] Make auto follow patterns work with security (#33501) + + Relates to #33007 + +commit 3046656ab11b0018729dc158d764b8f6246ec92d +Author: Ryan Ernst +Date: Sun Sep 16 19:18:00 2018 -0700 + + Scripting: Rework joda time backcompat (#33486) + + This commit switches the joda time backcompat in scripting to use + augmentation over ZonedDateTime. The augmentation methods provide + compatibility with the missing methods between joda's DateTime and + java's ZonedDateTime. Due to getDayOfWeek returning an enum in the java + API, ZonedDateTime is wrapped so that the method can return int like the + joda time does. The java time api version is renamed to + getDayOfWeekEnum, which will be kept through 7.x for compatibility while + users switch back to getDayOfWeek once joda compatibility is removed. + +commit e5d82c3dea8dd69889000ffa68ba67dbe3fdf03d +Author: Ryan Ernst +Date: Sun Sep 16 11:11:51 2018 -0700 + + Test: Fix dv date bwc tests when no docs have a value (#32798) + + This commit adds a guard around the rare case that no documents in the + 10 iterations actually have any values, thus making the warning check + incorrect. + + closes #32779 + +commit 770ad539780ad8edf24283b4996ba3661ad13c92 +Author: Jason Tedor +Date: Sun Sep 16 10:35:23 2018 -0400 + + Introduce long polling for changes (#33683) + + Rather than scheduling pings to the leader index when we are caught up + to the leader, this commit introduces long polling for changes. We will + fire off a request to the leader which if we are already caught up will + enter a poll on the leader side to listen for global checkpoint + changes. These polls will timeout after a default of one minute, but can + also be specified when creating the following task. We use these time + outs as a way to keep statistics up to date, to not exaggerate time + since last fetches, and to avoid pipes being broken. + +commit 069605bd91ffb6c4ea62c65de05cd294ff402f10 +Author: Jason Tedor +Date: Sun Sep 16 07:32:12 2018 -0400 + + Do not count shard changes tasks against REST tests (#33738) + + When executing CCR REST tests it is going to be expected after global + checkpoint polling goes in that shard changes tasks can still be pending + at the end of the test. One way to deal with this is to set a low + timeout on these polls, but then that means we are not executing our + REST tests with our default production settings and instead would be + using an unrealistic low timeout. Alternatively, since we expect these + tasks to be there, we can not count them against the test. That is what + this commit does. + +commit db40315afb9d491182b3f7c0745b985718ffd1d7 +Author: Dimitris Athanasiou +Date: Sun Sep 16 11:54:55 2018 +0100 + + [HLRC][ML] Add ML get datafeed API to HLRC (#33715) + + Relates #29827 + +commit 73417bf09afdfa6eeb7a7d620218b843e8d2653e +Author: Jason Tedor +Date: Sat Sep 15 10:18:59 2018 -0400 + + Move CCR REST tests to a sub-project of ccr + + This commit moves these REST tests (possibly temporarily) to a + sub-project of ccr. We do this (again, possibly temporarily) to keep + them within the ccr sub-project yet there are changes within 6.x that + prevent these from being in the top-level project (the cluster formation + tasks are trying to install x-pack-ccr into the + integ-test-zip). Therefore, we isolate these for now until we can + understand why there are differences between 6.x and master. + +commit aa56892f2fb8416e32bbcea3366e0c4e0146601d +Author: Jason Tedor +Date: Sat Sep 15 09:18:15 2018 -0400 + + Move CCR REST tests to ccr sub-project (#33731) + + This commit moves the CCR REST tests to the ccr sub-project as another + step towards running :x-pack:plugin:ccr:check giving us full coverage on + CCR. + +commit f037edb8e3a9d9e38bfe1cb376d178dc30c67416 +Author: Jason Tedor +Date: Sat Sep 15 09:16:33 2018 -0400 + + Move CCR monitoring tests to ccr sub-project (#33730) + + This commit moves the CCR monitoring tests from the monitoring + sub-project to the ccr sub-project. + +commit e6cbaa5a78f4dfad68df84d2e1af469946e3b179 +Merge: 1f048d3d3f8 92a48fa6b84 +Author: Lee Hinman +Date: Fri Sep 14 16:27:37 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 1f048d3d3f8b8bf5190587e203d74da6913beedf +Author: Lee Hinman +Date: Fri Sep 14 14:38:23 2018 -0600 + + Remove unneeded listener on MoveToNextStepUpdateTask (#33725) + + There was a listener that re-runs the policy with the new state when the cluster + state is processed by the `MoveToNextStepUpdateTask`. This removes this listener + as we will execute the policy through the `IndexLifecyleService` cluster state + listener. + +commit 92a48fa6b84a331f966dec800c9e44234f7a2b73 +Author: Jason Tedor +Date: Fri Sep 14 16:14:03 2018 -0400 + + Add script to cache dependencies (#33726) + + With the introduction of immutable workers into our CI, we now have a + problem where we would have to download dependencies on every single + build. To this end our infrastructure team has introduced the + possibility to download dependencies one time and then bake these into + the base immutable worker image. For this, we introduce our version of + this special script which runs a task that downloads all dependencies of + all configurations. With this script, our infrastructure team will be + rebuilding these images on an at-least daily basis. This helps us avoid + having to download the dependencies for every single build. + +commit 9706584836f19b1666da2d86fb12b425c3a667ed +Author: Lisa Cawley +Date: Fri Sep 14 13:09:47 2018 -0700 + + [DOCS] Moves security reference to docs folder (#33643) + +commit a0f0d7860ec4cf9881146b143fc034cef10c2fbd +Author: Jason Tedor +Date: Fri Sep 14 14:45:58 2018 -0400 + + Cleanup assertions in global checkpoint listeners (#33722) + + This commit is a cleanup of the assertions in global checkpoint + listeners, simplifying them and adding some messages to them in case the + assertions trip. + +commit 82a6ae1daed4f95e3841f5e40cf846c039780ca0 +Author: Martijn van Groningen +Date: Fri Sep 14 17:18:00 2018 +0200 + + [CCR] Move ccr tests in core module back to ccr module (#33711) + + When developing ccr it is not ideal if tests are in multiple modules. + Even the classes these tests test are in the core module, it is easier + if these tests are in ccr module in order to avoid running the test task + in core module. This results in running many non ccr tests. + + This way when developing ccr we can run locally: + ./gradlew x-pack:plugin:core:precommit x-pack:plugin:ccr:check + + before pushing to PR branches and be confident that the PR build passes, + without running x-pack:plugin:core:check task. + +commit b04faa059b50bfd8c55a5704c5bca432439ec85d +Author: David Kyle +Date: Fri Sep 14 15:00:18 2018 +0100 + + HLRC: ML PUT Calendar (#33362) + +commit bcbbbdf660cb1df4a846be39fecc879b73267e06 +Author: Christoph Büscher +Date: Fri Sep 14 15:52:47 2018 +0200 + + [Tests] Fix randomization in StringTermsIT (#33678) + + It looks like the COLLECT_SEGMENT_ORDS flag should be randomized. + +commit d9f5e4fd2e06c9b69f3b4744e49e747e1ff708b4 +Author: Ioannis Kakavas +Date: Fri Sep 14 13:49:35 2018 +0300 + + Pin TLS1.2 in SSLConfigurationReloaderTests + + Ensure that the SSLConfigurationReloaderTests can run with JDK 11 + by pinning the HttpClient to TLS version to TLS1.2. This is necessary + becase even if the MockWebServer is set to user TLS1.2, we don't + set its enabled protocols, so if it receives a TLS1.3 request (which + is the default behavior for HttpClient in JDK11), it will use TLS1.3 + and the original issue will manifest again. + + Relates #33127 + Resolves #32124 + +commit 39191331d19e00312f0de2db850fd72c92dafdee +Author: Jason Tedor +Date: Fri Sep 14 09:32:03 2018 -0400 + + Only notify ready global checkpoint listeners (#33690) + + When we add a global checkpoint listener, it is also carries along with + it a value that it thinks is the current global checkpoint. This value + can be above the actual global checkpoint on a shard if the listener + knows the global checkpoint from another shard copy (e.g., the primary), + and the current shard copy is lagging behind. Today we notify the + listener whenever the global checkpoint advances, regardless if it goes + above the current global checkpoint known to the listener. This commit + reworks this implementation. Rather than thinking of the value + associated with the listener as the current global checkpoint known to + the listener, we think of it as the value that the listener is waiting + for the global checkpoint to advance to (inclusive). Now instead of + notifying all waiting listeners when the global checkpoint advances, we + only notify those that are waiting for a value not larger than the + actual global checkpoint that we advanced to. + +commit 4f68104865bb5f36b473a223a88ca8e1ed4feff4 +Author: Adrien Grand +Date: Fri Sep 14 14:59:16 2018 +0200 + + Don't count hits via the collector if the hit count can be computed from index stats. (#33701) + + This is something that we were already doing when sorting by field, which is + now also done when sorting by score. As-is this change will speed up top-k + `term` queries. This could work for `match_all` queries as well when we + implement the `setMinCompetitiveScore` API on their Scorer. + +commit 2282150f3407d28a797a79df11ab7b6fc6c36203 +Author: Jason Tedor +Date: Fri Sep 14 08:52:46 2018 -0400 + + Expose retries for CCR fetch failures (#33694) + + This commit exposes the number of times that a fetch has been tried to + the CCR stats endpoint, and to CCR monitoring. + +commit 140c3bb61ce33b3d46aeb36114d5a0d05dd02b20 +Author: markharwood +Date: Fri Sep 14 13:46:36 2018 +0100 + + Test fix - Graph vertices could appear in different orders based on map insertion sequence (#33709) + + Solved by sorting the vertices arrays before comparison + Closes #33686 + +commit 31e8781eaa36d23d16ef3efcb12bb45b282d5785 +Merge: 1d18e2854cb 568ac10ca69 +Author: David Turner +Date: Fri Sep 14 14:28:28 2018 +0200 + + Merge branch 'master' into zen2 + +commit c86e2d5211e5fe8667f8bb716552e727ecc081b0 +Author: Albert Zaharovits +Date: Fri Sep 14 15:25:53 2018 +0300 + + Structured audit logging (#31931) + + Changes the format of log events in the audit logfile. + It also changes the filename suffix from `_access` to `_audit`. + The new entry format is consistent with Elastic Common Schema. + Entries are formatted as JSON with no nested objects and field + names have a dotted syntax. Moreover, log entries themselves + are not spaced by commas and there is exactly one entry per line. + In addition, entry fields are ordered, unlike a typical JSON doc, + such that a human would not strain his eyes over jumbled + fields from one line to the other; the order is defined in the log4j2 + properties file. + The implementation utilizes the log4j2's `StringMapMessage`. + This means that the application builds the log event as a map + and the log4j logic (the appender's layout) handle the format + internally. The layout, such as the set of printed fields and their + order, can be changed at runtime without restarting the node. + +commit faa3c1624136ba075c737bce312f570564b9ba35 +Author: Alexander Reelsen +Date: Fri Sep 14 13:55:16 2018 +0200 + + Core: Add DateFormatter interface for java time parsing (#33467) + + The existing approach used date formatters when a format based string + like `date_time||epoch_millis` was used, instead of the custom code. + + In order to properly solve this, a new interface called + `DateFormatter` has been added, which now can be implemented for custom + formatters. Currently there are two implementations, one using java time + and one doing the epoch_millis formatter, which simply parses a number + and then converts it to a date in UTC timezone. + + The DateFormatter interface now also has a method to retrieve the name + of the formatter pattern, which is needed for mapping changes anyway. + + The existing `CompoundDateTimeFormatter` class has been removed, the + name was not really nice anyway. + + One more minor change is the fact, that the new java time using + FormatDateFormatter does not try to parse the date with its printer + implementation first (which might be a strict one and fail), but a + printer can now be specified in addition. This saves one potential + failure/exception when parsing less strict dates. + + If only a printer is specified, the printer will also be used as a + parser. + +commit 222f42274ec7a8686e58fe4b7acff38a438a0776 +Author: Martijn van Groningen +Date: Fri Sep 14 13:28:11 2018 +0200 + + [CCR] Check whether the rejected execution exception has the shutdown flag set (#33703) + + and if so debug log it and otherwise rethrow. + + This should fix a couple of test failures where during test teardown tests + failed due to uncaught exceptions being detected. + +commit b8fb83d7a4663e7ed45a2f42ada826766f956f78 +Author: Igor Motov +Date: Fri Sep 14 14:17:27 2018 +0400 + + Mute ClusterDisruptionIT#testSendingShardFailure + + Tracked by #33704 + +commit f7c131f118a0408744578e2c38aea3389429c728 +Author: Jason Tedor +Date: Fri Sep 14 05:55:46 2018 -0400 + + Revert "Mute FullClusterRestartSettingsUpgradeIT" + + This reverts commit e9826164bddf0263f0a826e6d2824d5f103bb5b4. + +commit d65ff17b7edbed50e947e863d981a850de0736da +Author: Jason Tedor +Date: Wed Sep 12 17:05:00 2018 -0400 + + Adjust BWC version on settings upgrade test (#33650) + + The skip_unavailable setting did not exist until 6.1.0. This means that + we need to skip this test on versions prior to 6.1.0. We need to use + this setting because otherwise we will fail startup without it (since we + are not setting up a real remote cluster connection). This commit adds a + skip for all versions prior to 6.1.0. + +commit 568ac10ca696588dd2c4ce9ee2b0895643e4cef0 +Author: David Roberts +Date: Fri Sep 14 09:29:11 2018 +0100 + + [ML] Allow overrides for some file structure detection decisions (#33630) + + This change modifies the file structure detection functionality + such that some of the decisions can be overridden with user + supplied values. + + The fields that can be overridden are: + + - charset + - format + - has_header_row + - column_names + - delimiter + - quote + - should_trim_fields + - grok_pattern + - timestamp_field + - timestamp_format + + If an override makes finding the file structure impossible then + the endpoint will return an exception. + +commit 5f495c18dfd7cc8896b5a84868a843c0602d50e3 +Author: Jim Ferenczi +Date: Fri Sep 14 10:22:11 2018 +0200 + + Adapt skip version for doc_values format deprecation + + This commit fixes bwc rest tests for the doc_values format deprecation + in search. The message of the deprecation changed in 6.4.1 so the bwc test + should not check against 6.4.0. + +commit 4bcad95fe717714600cfe2d3d8109749ac0e00fc +Author: Martijn van Groningen +Date: Fri Sep 14 09:59:06 2018 +0200 + + [TEST] wait for no initializing shards + +commit c9131983f53d91d6161a9a1e7747c70b28dc0455 +Author: Christoph Büscher +Date: Fri Sep 14 09:41:20 2018 +0200 + + [Docs] Minor fix in `has_child` javadoc comment (#33674) + + The min and max constants are accidentaly the wrong way around. + +commit e9826164bddf0263f0a826e6d2824d5f103bb5b4 +Author: Igor Motov +Date: Fri Sep 14 10:14:02 2018 +0400 + + Mute FullClusterRestartSettingsUpgradeIT + + Tracked by #33697 + +commit d810f1b094e7629332d282ba3e403187ab7b17b3 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Sep 14 17:17:53 2018 +1000 + + [Kerberos] Add realm name & UPN to user metadata (#33338) + + We have a Kerberos setting to remove realm part from the user + principal name (remove_realm_name). If this is true then + the realm name is removed to form username but in the process, + the realm name is lost. For scenarios like Kerberos cross-realm + authentication, one could make use of the realm name to determine + role mapping for users coming from different realms. + This commit adds user metadata for kerberos_realm and + kerberos_user_principal_name. + +commit 8ae1eeb3039efae9048e92362bfbde91c95d48d4 +Author: Ioannis Kakavas +Date: Fri Sep 14 09:42:03 2018 +0300 + + [TESTS] Disable specific locales for RestrictedTrustManagerTest (#33299) + + Disable specific Thai and Japanese locales as Certificate expiration + validation fails due to the date parsing of BouncyCastle (that manifests + in a FIPS 140 JVM as this is the only place we use BouncyCastle). + Added the locale switching logic here instead of subclassing + ESTestCase as these are the only tests that fail for these locales and + JVM combination. + + Resolves #33081 + +commit 736053c6586fa4864e65423df06df25ce8b6949c +Author: Costin Leau +Date: Fri Sep 14 09:05:34 2018 +0300 + + SQL: Return functions in JDBC driver metadata (#33672) + + Update JDBC database metadata to advertised the supported functions + Add aliases to some date/time functions to match the ODBC spec + + Fix #33671 + +commit 0b4960ff6b93d94dae824156dddbf0084f8ea1b3 +Author: Armin Braun +Date: Fri Sep 14 06:21:18 2018 +0200 + + SCRIPTING: Move terms_set Context to its Own Class (#33602) + + * SCRIPTING: Move terms_set Context to its Own Class + + * Extracted TermsSetQueryScript + * Kept mechanics close to what they were with SearchScript + +commit 189aaceecf764d5ec7b34cd26e27ae6f621fe4cf +Author: Nhat Nguyen +Date: Thu Sep 13 22:15:21 2018 -0400 + + AwaitsFix testRestoreMinmal + + Tracked at #33689 + +commit d3e27ff2f62371cd93d14f61cd8a1bd881129f99 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Sep 14 10:07:19 2018 +1000 + + [Kerberos] Move tests based on SimpleKdc to evil-tests (#33492) + + We have a test dependency on Apache Mina when using SimpleKdcServer + for testing Kerberos. When checking for LDAP backend connectivity, + the code checks for deadlocks which require additional security + permissions accessClassInPackage.sun.reflect. As this is only for + test and we do not want to add security permissions to production, + this commit moves these tests and related classes to + x-pack evil-tests where they can run with security manager disabled. + The plan is to handle the security manager exception in the upstream issue + DIRMINA-1093 + and then once the release is available to run these tests with security + manager enabled. + + Closes #32739 + +commit 3914a980f799b39a0c2983fba9640ae50f8d2e4c +Author: Jay Modi +Date: Thu Sep 13 14:40:36 2018 -0600 + + Security: remove wrapping in put user response (#33512) + + This change removes the wrapping of the created field in the put user + response. The created field was added as a top level field in #32332, + while also still being wrapped within the `user` object of the + response. Since the value is available in both formats in 6.x, we can + remove the wrapped version for 7.0. + +commit 7dd22f09dcebb44da6e9f6aa22bdd2f7ceb78c9a +Author: Michael Basnight +Date: Thu Sep 13 15:17:37 2018 -0500 + + Mute failing JdbcSqlSpec functions + + Relates #33687 + +commit 32a22ca00e5329b785e4aa248677aab7d3349191 +Author: Costin Leau +Date: Thu Sep 13 22:07:23 2018 +0300 + + DOC: improved wording in SQL client app section + +commit 60ab4f97ab15c138412e2af10deba57a6b4e5e9d +Author: Costin Leau +Date: Thu Sep 13 21:55:44 2018 +0300 + + SQL: Return correct catalog separator in JDBC (#33670) + + JDBC DatabaseMetadata returns correct separator (:) for catalog/cluster names. + + Fix #33654 + +commit 53ba253aa4602158881814609872435e6d66feeb +Author: Martijn van Groningen +Date: Thu Sep 13 20:52:00 2018 +0200 + + [CCR] Add validation for max_retry_delay (#33648) + +commit b9d0c8f25cf6535e2f4583bc4cceaf44e5913501 +Author: Martijn van Groningen +Date: Thu Sep 13 20:47:24 2018 +0200 + + [CCR] Add monitoring mapping verification test (#33662) + + Added test that verifies that all fields in ShardFollowNodeTaskStatus are + mapped in monitoring-es.json + +commit 040695b64e6452c9e54109b9d31fe3d1efdd69f7 +Author: Armin Braun +Date: Thu Sep 13 20:45:48 2018 +0200 + + CORE: Disable Setting Type Validation (#33660) (#33669) + + * Reverts setting type validation introduced in #33503 + +commit e4eb631b8ec0820e6d016b4ff5fa50b3a351362e +Author: Jason Tedor +Date: Thu Sep 13 13:55:08 2018 -0400 + + Revert "Use serializable exception in GCP listeners (#33657)" + + This reverts commit 6dfe54c8381e2b9046de836fb07fabaa03a02452. + +commit 7e51b960fbc109b916dfb321eb05b44358a2bd17 +Author: Benjamin Trent +Date: Thu Sep 13 10:44:33 2018 -0700 + + Adding index refresh (#33647) + +commit c3a817957d6ff02d0412c37d28c916457e2d9531 +Author: Lisa Cawley +Date: Thu Sep 13 10:42:26 2018 -0700 + + [DOCS] Moves securing-communications to docs (#33640) + +commit 9600819cef27ea3dd7fe5fe93c228035e13bb320 +Author: Dimitris Athanasiou +Date: Thu Sep 13 17:13:36 2018 +0100 + + [HLRC][ML] Add ML delete datafeed API to HLRC (#33667) + + Relates #29827 + +commit dcbbaad296f6955bdb61cf461943c3fac2984d11 +Author: Nhat Nguyen +Date: Thu Sep 13 11:53:43 2018 -0400 + + Mute testRecoveryWithConcurrentIndexing + + Relates #33473 + Relates #33616 + +commit b3071133d4f66c1715d496c76909ff576619a65e +Author: Nhat Nguyen +Date: Thu Sep 13 11:18:03 2018 -0400 + + TEST: decrease logging level in the flush test + + Relates #31629 + +commit a192785fc84383aac8837a8467cc302dc0523ccf +Author: Costin Leau +Date: Sat Sep 8 16:23:43 2018 +0300 + + DOC: Add SQL section on client applications + + Add setup instructions for a number of GUI SQL applications + +commit d806a0e59d1cf0730c5f089b551385f667bad26e +Author: Jason Tedor +Date: Thu Sep 13 07:00:40 2018 -0400 + + Fix race in global checkpoint listeners test + + This race can occur if the latch from the listener notifies the test + thread and the test thread races ahead before the scheduler thread has a + chance to emit the log message. This commit fixes this test by not + counting down the latch until after the log message we are going to + assert on has been emitted. + +commit 6dfe54c8381e2b9046de836fb07fabaa03a02452 +Author: Jason Tedor +Date: Thu Sep 13 06:35:36 2018 -0400 + + Use serializable exception in GCP listeners (#33657) + + We used TimeoutException here but that's not serializable. This commit + switches to a serializable exception so that we can test for the + exception type on the remote side. + +commit a69ae6b89f0560b2dfc97e86eebd1268e945e2b9 +Author: Martijn van Groningen +Date: Thu Sep 13 11:36:52 2018 +0200 + + [CCR] Add metadata to keep track of the index uuid of the leader index in the follow index (#33367) + + The follow index api checks if the recorded uuid in the follow index matches + with uuid of the leader index and fails otherwise. This validation will + prevent a follow index from following an incompatible leader index. + + The create_and_follow api will automatically add this custom index metadata + when it creates the follow index. + + Closes #31505 + +commit 8e59de3eb2df9915e4878702fbd3edd55d019ffa +Merge: 5a349ec0e6f 7d3b99a9b79 +Author: Colin Goodheart-Smithe +Date: Thu Sep 13 09:46:14 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 7d3b99a9b7998997c54aa4678726e975d9395be7 +Author: Marios Trivyzas +Date: Thu Sep 13 10:28:05 2018 +0200 + + SQL: Fix result column names for CAST (#33604) + + Previously, when an non-pruned cast (casting as a different + data type) got applied on a table column in the `SELECT` clause, + the name of the result column didn't contain the target data type + of the cast, e.g.: + + SELECT CAST(MAX(salary) AS DOUBLE) FROM "test_emp" + + returned as column name: + + CAST(MAX(salary)) + + instead of: + + CAST(MAX(salary) AS DOUBLE) + + Closes #33571 + + * Added more tests for trivial casts that are pruned + +commit 6ca36bba15af11c30850a7401ec6e0f552296407 +Author: Jim Ferenczi +Date: Thu Sep 13 09:21:27 2018 +0200 + + Fix field mapping updates with similarity (#33634) + + This change fixes a bug introduced in 6.3 that prevents fields with an explicit + similarity to be updated. It also adds a test that checks this case for similarities + but also for analyzers since they could suffer from the same problem. + + Closes #33611 + +commit 5a3fd8e4e7c8d2721b3542ffa859c9245678d6c2 +Author: David Turner +Date: Thu Sep 13 07:37:15 2018 +0200 + + Use file-based discovery not MockUncasedHostsProvider (#33554) + + Today we use a special unicast hosts provider, the `MockUncasedHostsProvider`, + in many integration tests, to deal with the dynamic nature of the allocation of + ports to nodes. However #33241 allows us to use file-based discovery to achieve + the same goal, so the special test-only `MockUncasedHostsProvider` is no longer + required. + + This change removes `MockUncasedHostProvider` and replaces it with file-based + discovery in tests based on `EsIntegTestCase`. + +commit 94f6d4560dfc0898137c19d717d43907316c0857 +Author: David Turner +Date: Thu Sep 13 07:32:33 2018 +0200 + + Revert "Mark NamedDateTimeProcessorTests as @AwaitsFix" + + This reverts commit 44a80d61b287b48e4e2a5782bfdb008d552858aa. + +commit 44a80d61b287b48e4e2a5782bfdb008d552858aa +Author: David Turner +Date: Thu Sep 13 07:24:51 2018 +0200 + + Mark NamedDateTimeProcessorTests as @AwaitsFix + +commit b097eff34244d6b046cdadfec84e8fc9f1af6714 +Author: Nhat Nguyen +Date: Wed Sep 12 21:27:59 2018 -0400 + + Resync fails to notify on unavaiable exceptions (#33615) + + We fail to notify the resync listener if the resync replication hits a + shard unavailable exception. Moreover, we no longer need to swallow + these unavailable exceptions. + + Relates #28571 + Closes #33613 + +commit b2f78e5a47a9fba5b29542a2babbdae939483474 +Author: Shaunak Kashyap +Date: Wed Sep 12 15:56:13 2018 -0700 + + [CCR Monitoring] Only collect stats for specified indices (#33646) + + Follow up to #33617. Relates to #30086. + + As with all other per-index Monitoring collectors, the `CcrStatsCollector` should only collect stats for the indices the user wants to monitor. This list is controlled by the `xpack.monitoring.collection.indices` setting and defaults to all indices. + +commit eb715d52903b9ca1789f8bfdd7158fa53a2bfa90 +Author: Jason Tedor +Date: Wed Sep 12 17:35:06 2018 -0400 + + Add follower index to CCR monitoring and status (#33645) + + This commit adds the follower index to CCR shard follow task status, and + to monitoring. + +commit b5d8495789f7637f96bfba0c1bb528a21971a495 +Author: Martijn van Groningen +Date: Wed Sep 12 21:50:22 2018 +0200 + + [CCR] Add auto follow pattern APIs to transport client. (#33629) + +commit 20c6c9c542835f1620a495c54f2d84235f338fa7 +Author: Jay Modi +Date: Wed Sep 12 13:08:09 2018 -0600 + + Address license state update/read thread safety (#33396) + + This change addresses some issues regarding thread safety around + updates and method calls on the XPackLicenseState object. There exists + a possibility that there could be a concurrent update to the + XPackLicenseState when there is a scheduled check to see if the license + is expired and a cluster state update. In order to address this, the + update method now has a synchronized block where member variables are + updated. Each method that reads these variables is now also + synchronized. + + Along with the above change, there was a consistency issue around + security calls to the license state. The majority of security checks + make two calls to the license state, which could result in incorrect + behavior due to the checks being made against different license states. + The majority of this behavior was introduced for 6.3 with the inclusion + of x-pack in the default distribution. In order to resolve the majority + of these cases, the `isSecurityEnabled` method is no longer public and + the logic is also included in individual methods about security such as + `isAuthAllowed`. There were a few cases where this did not remove + multiple calls on the license state, so a new method has been added + which creates a copy of the current license state that will not change. + Callers can use this copy of the license state to make decisions based + on a consistent view of the license state. + +commit 9b8fe85edb81378f16d3f0819b4f5604bad7e7cf +Author: Jason Tedor +Date: Wed Sep 12 14:38:24 2018 -0400 + + Remove volatile from global checkpoint listeners (#33636) + + This field does not need to be volatile because all accesses are done + under a lock. This commit removes the unnecessary volatile modifier from + this field. + +commit cfc20825ab8dea80d2e9dd3b5ff04a151602d8c1 +Author: Vladimir Dolzhenko +Date: Wed Sep 12 20:34:56 2018 +0200 + + mute NamedDateTimeProcessorTests + +commit 5fa81310cc1ce4506f93c984791a586a01fde4a9 +Author: Martijn van Groningen +Date: Wed Sep 12 19:42:00 2018 +0200 + + [CCR] Added history uuid validation (#33546) + + For correctness we need to verify whether the history uuid of the leader + index shards never changes while that index is being followed. + + * The history UUIDs are recorded as custom index metadata in the follow index. + * The follow api validates whether the current history UUIDs of the leader + index shards are the same as the recorded history UUIDs. + If not the follow api fails. + * While a follow index is following a leader index; shard follow tasks + on each shard changes api call verify whether their current history uuid + is the same as the recorded history uuid. + + Relates to #30086 + + Co-authored-by: Nhat Nguyen + +commit c023f67c5d843bb951dd2473f33806dff06819eb +Author: Jason Tedor +Date: Wed Sep 12 13:37:11 2018 -0400 + + Add migration note for remote cluster settings (#33632) + + The remote cluster settings search.remote.* have been renamed to + cluster.remote.* and are automatically upgraded in the cluster state on + gateway recovery, and on put. This commit adds a note to the migration + docs for these changes. + +commit 901d8035d9243d2136944809ef65087e6dbc652c +Author: Martijn van Groningen +Date: Wed Sep 12 19:36:17 2018 +0200 + + [CCR] Update es monitoring mapping and (#33635) + + * [CCR] Update es monitoring mapping and + change qa tests to query based on leader index. + + + Co-authored-by: Jason Tedor + +commit c783488e971d3099a2fc03446c978fae0286cba8 +Author: Simon Willnauer +Date: Wed Sep 12 17:47:10 2018 +0200 + + Add `_source`-only snapshot repository (#32844) + + This change adds a `_source` only snapshot repository that allows to wrap + any existing repository as a _backend_ to snapshot only the `_source` part + including live docs markers. Snapshots taken with the `source` repository + won't include any indices, doc-values or points. The snapshot will be reduced in size and + functionality such that it requires full re-indexing after it's successfully restored. + + The restore process will copy the `_source` data locally starts a special shard and engine + to allow `match_all` scrolls and searches. Any other query, or get call will fail with and unsupported operation exception. The restored index is also marked as read-only. + + This feature aims mainly for disaster recovery use-cases where snapshot size is + a concern or where time to restore is less of an issue. + + **NOTE**: The snapshot produced by this repository is still a valid lucene index. This change doesn't allow for any longer retention policies which is out of scope for this change. + +commit 141c6ef93ea9df206d25ba874284a7fe58a853d4 +Author: Vladimir Dolzhenko +Date: Wed Sep 12 17:18:34 2018 +0200 + + upgrade randomizedrunner to 2.7.0 (#33623) + +commit fe478c23b797f5bbde8b8db4a177247c5b538424 +Author: Christoph Büscher +Date: Wed Sep 12 16:56:03 2018 +0200 + + [Docs] Fix heading in composite-aggregation.asciidoc (#33627) + + The heading for the "Missing buckets" should be on the same level + as the the "Order" section. + +commit 36ba3cda7eeb9646f917a0e4338b1eaec9a2b92d +Author: Jason Tedor +Date: Wed Sep 12 10:53:22 2018 -0400 + + Enable global checkpoint listeners to timeout (#33620) + + In cross-cluster replication, we will use global checkpoint listeners to + long poll for updates to a shard. However, we do not want these polls to + wait indefinitely as it could be difficult to discern if the listener is + still waiting for updates versus something has gone horribly wrong and + cross-cluster replication is stuck. Instead, we want these listeners to + timeout after some period (for example, one minute) so that they are + notified and we can update status on the following side that + cross-cluster replication is still active. After this, we will + immediately enter back into a poll mode. + + To do this, we need the ability to associate a timeout with a global + checkpoint listener. This commit adds this capability. + +commit bcac7f5e55fcd9a3c3ed0c6b8772dbb57f681a1a +Author: Tanguy Leroux +Date: Wed Sep 12 16:03:52 2018 +0200 + + Fix checkstyle violation in ShardFollowNodeTask + +commit 2eb2313b6031da8b933c95e785545938c63eface +Author: Dimitris Athanasiou +Date: Wed Sep 12 14:52:36 2018 +0100 + + [HLRC][ML] Add ML put datafeed API to HLRC (#33603) + + This also changes both `DatafeedConfig` and `DatafeedUpdate` + to store the query and aggs as a bytes reference. This allows + the client to remove its dependency to the named objects + registry of the search module. + + Relates #29827 + +commit 7e195c2912b9ed9fad3099761562dd0a7731635c +Author: Tanguy Leroux +Date: Wed Sep 12 15:27:57 2018 +0200 + + Update AWS SDK to 1.11.406 in repository-s3 (#30723) + +commit 23f12e42c13878b0ff3b4c33420eb1570ee586cd +Author: Jason Tedor +Date: Wed Sep 12 09:13:07 2018 -0400 + + Expose CCR stats to monitoring (#33617) + + This commit exposes the CCR stats endpoint to monitoring collection. + + Co-authored-by: Martijn van Groningen + +commit 0b567c0eeba3ad4c89e349f7f5439292a1f4356f +Author: Joel Green +Date: Wed Sep 12 13:34:05 2018 +0100 + + [Docs] Update match-query.asciidoc (#33610) + +commit d9bbb89b2611c8edcba7c81bde67578241af6136 +Author: Nhat Nguyen +Date: Wed Sep 12 08:24:11 2018 -0400 + + TEST: Adjust rollback condition when shard is empty + + If a shard is empty, it won't rollback its engine on promotion. + This commit adjusts the expectation in the rollback test. + + Relates #33473 + +commit 96c49e5ed066bcaa82392c1acdced94ca43b28bd +Author: Martijn van Groningen +Date: Wed Sep 12 12:49:51 2018 +0200 + + [CCR] Improve shard follow task's retryable error handling (#33371) + + Improve failure handling of retryable errors by retrying remote calls in + a exponential backoff like manner. The delay between a retry would not be + longer than the configured max retry delay. Also retryable errors will be + retried indefinitely. + + Relates to #30086 + +commit c92ec1c5d7c516b83e6264528e88ce42e8f6913e +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Wed Sep 12 09:16:40 2018 +0200 + + Forbid negative `weight` in Function Score Query (#33390) + + This change forbids negative `weight` in Function Score query. Negative scores are forbidden in Lucene 8. + +commit 4561c5ee832d7a7dfe0ad3aeea3ff60ef0340fd8 +Author: Jim Ferenczi +Date: Wed Sep 12 08:47:32 2018 +0200 + + Clarify context suggestions filtering and boosting (#33601) + + This change clarifies the documentation of the context completion suggester + regarding filtering and boosting with contexts. + Unlike the suggester v1, filtering on multiple contexts + works as a disjunction, a suggestion matches if it contains at least one of the provided + context values and boosting selects the maximum score among the matching contexts. + This commit also adapts an old test that was written for the v1 suggester and commented out + for version 2 because the behavior changed. + +commit 20476b9e062e57b73f4ddf14e889d4de6e2f9c88 +Author: Jason Tedor +Date: Wed Sep 12 01:54:34 2018 -0400 + + Disable CCR REST endpoints if CCR disabled (#33619) + + This commit avoids enabling the CCR REST endpoints if CCR is disabled. + +commit dcdacd2f3fef8eb803cfd3256b9c7f83eb486e95 +Author: Jason Tedor +Date: Wed Sep 12 01:19:05 2018 -0400 + + Lower version on full cluster restart settings test + + The change to upgrade cross-cluster search settings was backported to + 6.5.0. Therefore, this assumption needs to be reduced to the latest + version where settings were not automatically upgraded. + +commit c74c46edc3c583d09e88ac5d56352fdccdedf5ab +Author: Jason Tedor +Date: Wed Sep 12 01:14:43 2018 -0400 + + Upgrade remote cluster settings (#33537) + + This commit adds settings upgraders for the search.remote.* settings + that can be in the cluster state to automatically upgrade these settings + to cluster.remote.*. Because of the infrastructure that we have here, + these settings can be upgraded when recovering the cluster state, but + also when a user tries to make a dynamic update for these settings. + +commit 94cdf0cebacde4188e6ca5f84843d732aae8cf7c +Author: Armin Braun +Date: Wed Sep 12 06:15:36 2018 +0200 + + NETWORKING: http.publish_host Should Contain CNAME (#32806) + + * NETWORKING: http.publish_host Should Contain CNAME + + * Closes #22029 + +commit 9752540866ee9fee3946b2b0c7a0021855670ab4 +Author: Jason Tedor +Date: Tue Sep 11 23:17:26 2018 -0400 + + Add test coverage for global checkpoint listeners + + This commit adds test coverage for two cases not previously covered by + the existing testing. Namely, we add coverage ensuring that the executor + is used to notify listeners being added that are immediately notified + because the shard is closed or because the global checkpoint is already + beyond what the listener knows. + +commit 743327efc204c3e3c7e26567f285c5c5032f30a7 +Author: Nhat Nguyen +Date: Tue Sep 11 22:09:37 2018 -0400 + + Reset replica engine to global checkpoint on promotion (#33473) + + When a replica starts following a newly promoted primary, it may have + some operations which don't exist on the new primary. Thus we need to + throw those operations to align a replica with the new primary. This can + be done by first resetting an engine from the safe commit, then replaying + the local translog up to the global checkpoint. + + Relates #32867 + +commit 27e07ec859f392bf48280eb6d8fca9f88c4e8058 +Author: Benjamin Trent +Date: Tue Sep 11 16:32:52 2018 -0700 + + HLRC: ML Delete Forecast API (#33526) + + * HLRC: ML Delete Forecast API + +commit 9f8dff9281e5ff16c9e819944125174a0160d17b +Author: Jason Tedor +Date: Tue Sep 11 19:11:30 2018 -0400 + + Remove debug logging in full cluster restart tests (#33612) + + These logs are incredibly verbose, and it makes chasing normal failures + burdensome. This commit removes the debug logging, which can be + reenabled again if needed. + +commit eca37e6e0aa8b68096e86e8823626c43b9116328 +Author: Jason Tedor +Date: Tue Sep 11 16:37:52 2018 -0400 + + Expose CCR to the transport client (#33608) + + This commit exposes CCR to the transport client. + +commit 1e577d3ce8daa81530999426adddd9845332454d +Author: Nhat Nguyen +Date: Tue Sep 11 16:22:59 2018 -0400 + + Mute testIndexDeletionWhenNodeRejoins + + Tracked at #33613 + +commit 91bca174f507b56e320ab2bd6bb0afc54e52e71b +Author: Costin Leau +Date: Tue Sep 11 18:47:49 2018 +0300 + + SQL: Make Literal a NamedExpression (#33583) + + * SQL: Make Literal a NamedExpression + + Literal now is a NamedExpression reducing the need for Aliases for + folded expressions leading to simpler optimization rules. + + Fix #33523 + +commit cbc6fa0ecb5cd6fe8695dee5ae93b2a5325e69c4 +Author: Lisa Cawley +Date: Tue Sep 11 07:56:26 2018 -0700 + + [DOCS] Adds missing built-in user information (#33585) + +commit 624b84f897bff2fc2570dcba875b43f16d8d1321 +Author: Colin Goodheart-Smithe +Date: Tue Sep 11 14:32:43 2018 +0100 + + Improves doc values format deprecation message (#33576) + + * Improves doc values format deprecation message + + This changes the deprecation message when doc values fields do not + supply a format form logging a deprecation warning for each offending + field individually to logging a single message which lists all + offending fields + + Closes #33572 + + * Updates YAML test with new deprecation message + + Also adds a test to ensure multiple deprecation warnings are collated + into one message + + * Condenses collection of fields without format check + + Moves the collection of fields that don't have a format to a separate + loop and moves the logging of the deprecation warning to be next to it + at the expesnse of looping through the field list twice + + * fixes typo + + * Fixes test + +commit 73c75bef216ee2d0658c7029c7c97e587983caad +Author: Jason Tedor +Date: Tue Sep 11 08:40:22 2018 -0400 + + Preserve cluster settings on full restart tests (#33590) + + Today the full cluster restart tests do not preserve cluster settings on + restart. This is a mistake because it is not an accurate reflection of + reality, we do not expect users to clear cluster settings when they + perform a full cluster restart. This commit makes it so that all full + cluster restart tests preserve settings on upgrade. + +commit 36bdad4895395e6854ffe2b783aec47ec93b4b1f +Author: Alan Woodward +Date: Tue Sep 11 13:38:44 2018 +0100 + + Use IndexWriter.getFlushingBytes() rather than tracking it ourselves (#33582) + + Currently we keep track of how many bytes are currently being written to disk + in an AtomicLong within InternalEngine, updating it on refresh. The IndexWriter + has its own accounting for this, and exposes it via a getFlushingBytes method + in the latest lucene 8 snapshot. This commit removes the InternalEngine tracking + in favour of just using the IndexWriter method. + +commit ad4b5e427004351ae7e88dadfe4210db437ab764 +Author: Jason Tedor +Date: Tue Sep 11 08:35:42 2018 -0400 + + Fix upgrading of list settings (#33589) + + Upgrading list settings is broken because of the conversion that we do + to strings, and then when we try to put back the upgraded value we do + not know that it is a representation of a list. This commit addresses + this by adding special handling for list settings. + +commit 517cfc3cc0c2278a1287cf961c7db513e81dcbb6 +Author: Simon Willnauer +Date: Tue Sep 11 14:05:14 2018 +0200 + + Add read-only Engine (#33563) + + This change adds an engine implementation that opens a reader on an + existing index but doesn't permit any refreshes or modifications + to the index. + + Relates to #32867 + Relates to #32844 + +commit 2f3b542d57a6a1e50ba67e4c69b9dece0cac180b +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Tue Sep 11 12:48:14 2018 +0100 + + HLRC: Add ML get categories API (#33465) + + HLRC: Adding the ML 'get categories' API + +commit a3e1f1e46f444e656f1eed62736c85e0c2d903e6 +Author: Andrei Stefan +Date: Tue Sep 11 14:35:34 2018 +0300 + + SQL: Adds MONTHNAME, DAYNAME and QUARTER functions (#33411) + + * Added monthname, dayname and quarter functions + * Updated docs tests with the new functions + +commit 5a349ec0e6f28d049637fd4916f9baf51ef3089a +Author: Colin Goodheart-Smithe +Date: Tue Sep 11 10:18:24 2018 +0100 + + Renames lifecycle to policy in rest-api-spec (#33507) + + * Renames lifecycle to policy in rest-api-spec + + * Fix test + +commit f598297f55fa60df9fdefa4f34574d485b600b20 +Author: Alan Woodward +Date: Tue Sep 11 09:16:39 2018 +0100 + + Add predicate_token_filter (#33431) + + This allows users to filter out tokens from a TokenStream using painless scripts, + instead of having to write specialised Java code and packaging it up into a plugin. + + The commit also refactors the AnalysisPredicateScript.Token class so that it wraps + and makes read-only an AttributeSource. + +commit a55fa4fd6b145ce9da849834df9abe19307019f5 +Author: Andrei Stefan +Date: Tue Sep 11 11:00:56 2018 +0300 + + Fix Replace function. Adds more tests to all string functions. (#33478) + +commit 1d18e2854cbc64701cb68952d5053f9b62cceae3 +Author: David Turner +Date: Tue Sep 11 09:55:52 2018 +0200 + + Fix merge + +commit 8e05ce567fe734ed4d9eb299d5f4a972a200fcf9 +Author: David Roberts +Date: Tue Sep 11 08:46:26 2018 +0100 + + [ML] Rename input_fields to column_names in file structure (#33568) + + This change tightens up the meaning of the "input_fields" field + in the file structure finder output. Previously it was permitted + but not calculated for JSON and XML files. Following this change + the field is called "column_names" and is only permitted for + delimited files. + + Additionally the way the column names are set for headerless + delimited files is refactored to encapsulate the way they're + named to one line of the code rather than having the same + logic in two places. + +commit a2cd8f731ed494c16e1ecc7281843b1b76de6811 +Merge: a0d32f59475 ea3fdc90c69 +Author: David Turner +Date: Tue Sep 11 09:38:10 2018 +0200 + + Merge branch 'master' into zen2 + +commit ea3fdc90c69fdef18521bfdd522574bab4dea0cb +Author: Jason Tedor +Date: Mon Sep 10 20:06:42 2018 -0400 + + Add full cluster restart base class (#33577) + + This commit adds a base class for full cluster restart tests. + +commit 6075e159e5e47e7ed38073e417feb8a818dde259 +Author: Armin Braun +Date: Tue Sep 11 01:24:18 2018 +0200 + + Validate list values for settings (#33503) + + When we see a settings value, it could be a list. Yet this should only + happen if the underlying setting type is a list setting type. This + commit adds validation that when we get a setting value that is a list, + that the setting that we are getting is a list setting. And similarly, + if we get a value for a list setting, the underlying value should be a + list. + +commit 624b6bb487c6f5901aff8ee4213fea6f68bbd902 +Author: Nhat Nguyen +Date: Mon Sep 10 17:38:58 2018 -0400 + + Copy and validatie soft-deletes setting on resize (#33517) + + This change copies and validates the soft-deletes setting during resize. + If the source enables soft-deletes, the target must also enable it. + + Closes #33321 + +commit b6d68bd805f1858a0210e381402236dea1d42509 +Author: Nik Everett +Date: Mon Sep 10 17:03:52 2018 -0400 + + Test: Fix package name + + I created a test a few days ago and declared a package that doesn't line + up with the directory structure. Oops. I a little surprised nothing + complained. But this fixes it. + +commit f5a366a7f03c97a4c2a3d2b5d9cd697acd11a3ac +Author: Colin Goodheart-Smithe +Date: Mon Sep 10 21:31:41 2018 +0100 + + Fixing merge problem + +commit cdc4f57a773b588fe872f247ee62c3d5539e2faa +Merge: b7649fce0cd f441bb87ad9 +Author: Colin Goodheart-Smithe +Date: Mon Sep 10 21:30:44 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit f441bb87ad9d9089a9a826d0fa9c7eb64ce4faf6 +Author: Marios Trivyzas +Date: Mon Sep 10 22:02:11 2018 +0200 + + SQL: Fix result column names for arithmetic functions (#33500) + + Previously, when an arithmetic function got applied on a + table column in the `SELECT` clause, the name of the result + column contained weird characters used internally when + processing the SQL statement e.g.: + + SELECT CHAR(emp_no % 10000) FROM "test_emp" + + returned: + + CHAR((emp_no{f}#14) % 10000)) + + as the column name instead of: + + CHAR((emp_no) % 10000)) + + Also, fix an issue that causes a ClassCastException to be thrown + when using functions where both arguments are literals. + + Closes #31869 + Closes #33461 + +commit 39c3234c2f3f1331a1a8cffc547c0998590e1598 +Author: Alan Woodward +Date: Mon Sep 10 20:51:55 2018 +0100 + + Upgrade to latest Lucene snapshot (#33505) + + * LeafCollector.setScorer() now takes a Scorable + * Scorers may not have null Weights + * IndexWriter.getFlushingBytes() reports how much memory is being used by IW threads writing to disk + +commit 5f4244755e9670d3ffb660827597e2deac76c2e2 +Author: Jason Tedor +Date: Mon Sep 10 14:25:30 2018 -0400 + + Enable not wiping cluster settings after REST test (#33575) + + In some cases we want to skip wiping cluster settings after a REST + test. For example, one use-case would be in the full cluster restart + tests where want to test cluster settings before and after a full + cluster restart. If we wipe the cluster settings before the restart, + then it would not be possible to assert on them after the restart. + +commit 9a2c77d1c363b83bf423fc567bcac4cd72812f0f +Author: Armin Braun +Date: Mon Sep 10 18:56:21 2018 +0200 + + MINOR: Remove Dead Code in SearchScript (#33569) + + * `lookup` is not used anywhere + * `getLeafContext` is not used anywhere + +commit 079d130d8c4827426ee15e7d7964b1c993a6c491 +Author: Tanguy Leroux +Date: Mon Sep 10 18:29:00 2018 +0200 + + [Test] Remove duplicate method in TestShardRouting (#32815) + +commit 74d41857c67a9dac0c94adeb0f61b609f63ed1ee +Author: Martijn van Groningen +Date: Mon Sep 10 16:46:15 2018 +0200 + + mute test on windows + + Relates #33570 + +commit 369db8a9d66950cd20ad634d9bff24f6578a6b16 +Author: Chris Roberson +Date: Mon Sep 10 08:50:07 2018 -0500 + + Update beats template to include apm-server metrics (#33286) + +commit 80c4661d0c90fec91b072799c8d9dd34700cb8bb +Author: Tanguy Leroux +Date: Mon Sep 10 14:20:04 2018 +0200 + + Fix typos (#33499) + +commit 8eebca32d2b3882d9f6c7bced4320f84b88f6d57 +Author: Martijn van Groningen +Date: Mon Sep 10 13:23:02 2018 +0200 + + [CCR] Delay auto follow license check (#33557) + + * [CCR] Delay auto follow license check + so that we're sure that there are auto follow patterns configured + + Otherwise we log a warning in case someone is running with basic or gold + license and has not used the ccr feature. + +commit c4adcee3ea3e224bf690cc9e9cbfc764b430c41c +Author: Martijn van Groningen +Date: Mon Sep 10 13:08:20 2018 +0200 + + [CCR] Add create_follow_index privilege (#33559) + + This is a new index privilege that the user needs to have in the follow cluster. + This privilege is required in addition to the `manage_ccr` cluster privilege in + order to execute the create and follow api. + + Closes #33555 + +commit 284c45a6ffb1a4eb2ce7498db25d362cae2c0c02 +Author: David Turner +Date: Mon Sep 10 11:23:05 2018 +0200 + + Strengthen FilterRoutingTests (#33149) + + Today the FilterRoutingTests take the belt-and-braces approach of excluding + some node attribute values and including some others. This means that we don't + really test that both inclusion and exclusion work correctly: as long as one of + them works as expected then the test will pass. This change improves these + tests by only using one approach at once, demonstrating that both do indeed + work, and adds tests for various other scenarios too. + +commit 77aeeda27530cb43206f8a6ce631ee71fa702883 +Author: Ioannis Kakavas +Date: Mon Sep 10 11:18:44 2018 +0300 + + Correctly handle PKCS#11 tokens for system keystore (#33460) + + * Correctly handle NONE keyword for system keystore + + As defined in the PKCS#11 reference guide + https://docs.oracle.com/javase/8/docs/technotes/guides/security/p11guide.html + PKCS#11 tokens can be used as the JSSE keystore and truststore and + the way to indicate this is to set `javax.net.ssl.keyStore` and + `javax.net.ssl.trustStore` to `NONE` (case sensitive). + + This commits ensures that we honor this convention and do not + attempt to load the keystore or truststore if the system property is + set to NONE. + + * Handle password protected system truststore + + When a PKCS#11 token is used as the system truststore, we need to + pass a password when loading it, even if only for reading + certificate entries. This commit ensures that if + `javax.net.ssl.trustStoreType` is set to `PKCS#11` (as it would + when a PKCS#11 token is in use) the password specified in + `javax.net.ssl.trustStorePassword` is passed when attempting to + load the truststore. + + Relates #33459 + +commit 95100e05f97cc56e4fa30d1a7db3f9054885a4c0 +Author: Julie Tibshirani +Date: Sun Sep 9 20:02:35 2018 -0700 + + Remove some duplicate request conversion methods. (#33538) + +commit e6ca55bca6942ca4bc0c892a81fec01071b13796 +Author: Nhat Nguyen +Date: Sun Sep 9 21:34:11 2018 -0400 + + Adjust bwc for stale primary recovery source (#33432) + + Relates #33432 + +commit 6bb817004b9d3f4cc9e723f0df50b5a2f1cdc064 +Author: Jason Tedor +Date: Sun Sep 9 20:49:19 2018 -0400 + + Add infrastructure to upgrade settings (#33536) + + In some cases we want to deprecate a setting, and then automatically + upgrade uses of that setting to a replacement setting. This commit adds + infrastructure for this so that we can upgrade settings when recovering + the cluster state, as well as when such settings are dynamically applied + on cluster update settings requests. This commit only focuses on cluster + settings, index settings can build on this infrastructure in a + follow-up. + +commit fcb15b0ce30e1a46d42f65874c1e4a91e2c3794c +Author: Dimitris Athanasiou +Date: Sun Sep 9 22:53:03 2018 +0100 + + [ML] Get job stats request should filter non-ML job tasks (#33516) + + When requesting job stats for `_all`, all ES tasks are accepted + resulting to loads of cluster traffic and a memory overhead. + This commit correctly filters out non ML job tasks. + + Closes #33515 + +commit d1b99877fad4af7473e18e46cc461d3901b39921 +Author: Jason Tedor +Date: Sun Sep 9 14:42:49 2018 -0400 + + Remove underscore from auto-follow API (#33550) + + This commit removes the leading underscore from _auto_follow in the + auto-follow API endpoints. + +commit 902d20cbbe7fa8a8fc9fe2408ad8d5711a4fc78e +Author: Nhat Nguyen +Date: Sun Sep 9 13:18:30 2018 -0400 + + CCR: Use single global checkpoint to normalize range (#33545) + + We may use different global checkpoints to validate/normalize the range + of a change request if the global checkpoint is advanced between these + calls. If this is the case, then we generate an invalid request range. + +commit d4b212c4c9f8f153522ef4c1d8f0c588bb5fc3d6 +Author: Armin Braun +Date: Sun Sep 9 17:31:02 2018 +0200 + + CORE: Make Pattern Exclusion Work with Aliases (#33518) + + * CORE: Make Pattern Exclusion Work with Aliases + + * Adds the pattern exclusion logic to finding aliases + * Closes #33395 + +commit 6eca6274090f8a460d9381ef27d6f335bfdd738b +Author: Jason Tedor +Date: Sun Sep 9 10:22:22 2018 -0400 + + Reverse logic for CCR license checks (#33549) + + This commit reverses the logic for CCR license checks in a few + actions. This is done so that the successful case, which tends to be a + larger block of code, does not require indentation. + +commit edc492419be3a30550f4b5816d438867d7a7c3c0 +Author: Jason Tedor +Date: Sun Sep 9 09:52:40 2018 -0400 + + Add latch countdown on failure in CCR license tests (#33548) + + We have some listeners in the CCR license tests that invoke Assert#fail + if the onSuccess method for the listener is unexpectedly invoked. This + can leave the main test thread hanging until the test suite times out + rather than failing quickly. This commit adds some latch countdowns so + that we fail quickly if these cases are hit. + +commit 9073dbefd6d5a1441ac11e7990e181fa74f3ce15 +Author: S.Y. Wang +Date: Sun Sep 9 20:47:47 2018 +0900 + + HLRC: Add put stored script support to high-level rest client (#31323) + + Relates to #27205 + +commit c67b0ba33edd361b49e2915d04ecf74e7e81f4d5 +Author: Jason Tedor +Date: Sun Sep 9 07:16:56 2018 -0400 + + Create temporary directory if needed in CCR test + + In the multi-cluster-with-non-compliant-license tests, we try to write + out a java.policy to a temporary directory. However, if this temporary + directory does not already exist then writing the java.policy file will + fail. This commit ensures that the temporary directory exists before we + attempt to write the java.policy file. + +commit 5a38c930fcc42d93475596c92903123e06613bac +Author: Jason Tedor +Date: Sun Sep 9 07:06:55 2018 -0400 + + Add license checks for auto-follow implementation (#33496) + + This commit adds license checks for the auto-follow implementation. We + check the license on put auto-follow patterns, and then for every + coordination round we check that the local and remote clusters are + licensed for CCR. In the case of non-compliance, we skip coordination + yet continue to schedule follow-ups. + +commit b7649fce0cd19225df7f94e34f107f3af3abe7ac +Author: Lee Hinman +Date: Sat Sep 8 21:40:55 2018 -0600 + + Rename "after" to "minimum_age" in lifecycle definition (#33530) + + This renames the "after" field to better reflect what the meaning is. + + Supercedes #32624 + +commit 94e4cb64c2480141fd54e19f64291ad7cd870bef +Author: Nhat Nguyen +Date: Sat Sep 8 19:29:31 2018 -0400 + + Bootstrap a new history_uuid when force allocating a stale primary (#33432) + + This commit ensures that we bootstrap a new history_uuid when force + allocating a stale primary. A stale primary should never be the source + of an operation-based recovery to another shard which exists before the + forced-allocation. + + Closes #26712 + +commit f27c3dcf881d8dc426d6d58afb63fb3774f45479 +Author: Armin Braun +Date: Sat Sep 8 10:18:45 2018 +0200 + + INGEST: Remove Outdated TODOs (#33458) + + * CompoundProcessor is in the ingest package now + -> resolved + * Java generics don't offer type checking so nothing + can be done here -> remvoed TODO and test + * #16019 was closed and not acted on + -> todo can go away + +commit ab9e2cddf169be536065e9b32d367fee978bbb0c +Author: Nik Everett +Date: Fri Sep 7 21:54:38 2018 -0400 + + Logging: Clean up skipping test + + Clean up on top of the last fix: if we skip the entire test case then + the test run would fail because we skipped all the tests. This adds a + dummy test case to prevent that. It is a fairly nasty work around I plan + to work on something that makes this not required any more anyway. + +commit 97736ac46a54073cf7081e9ed08bccbdac79ea6a +Author: Nik Everett +Date: Fri Sep 7 21:42:58 2018 -0400 + + Logging: Skip test if it'd fail + + If we're running on a platform where we can't install syscall filters + Elasticsearch logs a message before it reads the data directory to get + the node name. Because that log message doesn't have a node name this + test will fail. Since we mostly run the test on OSes where we *can* + install the syscall filters we can fairly safely skip the test on OSes + where we can't install the syscall filters. + + Closes #33540 + +commit 0b62d6bb9d7a441ef6802d43dcd4e2452961498f +Author: Nik Everett +Date: Fri Sep 7 21:27:59 2018 -0400 + + CRUD: AwaitsFix entire wait_for_refresh close test + + I disabled one branch a few hours ago because it failed in CI. It looks + like other branches can also fail so I'll disable them as well and look + more closely on Monday. + +commit facec187bbee24e3c3c43547e5d785191f2fd6d5 +Author: Jack Conradson +Date: Fri Sep 7 18:16:07 2018 -0700 + + Painless: Add Imported Static Method (#33440) + + Allow static methods to be imported in Painless and called using just the method name. + +commit 9a404f3def3f2b54e28cf8c15ff8ec1de1a25672 +Author: Jason Tedor +Date: Fri Sep 7 20:09:53 2018 -0400 + + Include fallback settings when checking dependencies (#33522) + + Today when checking settings dependencies, we do not check if fallback + settings are present. This means, for example, that if + cluster.remote.*.seeds falls back to search.remote.*.seeds, and + cluster.remote.*.skip_unavailable and search.remote.*.skip_unavailable + depend on cluster.remote.*.seeds, and we have set search.remote.*.seeds + and search.remote.*.skip_unavailable, then validation will fail because + it is expected that cluster.ermote.*.seeds is set here. This commit + addresses this by also checking fallback settings when validating + dependencies. To do this, we adjust the settings exist method to also + check for fallback settings, a case that it was not handling previously. + +commit 6b780e9926b33d2920be22005ca414cf9819e838 +Author: lcawl +Date: Fri Sep 7 16:53:36 2018 -0700 + + [DOCS] Fixing formatting issues in breaking changes + +commit 008e64b3955f48bae303d2f4028bad4968ba5578 +Author: Gordon Brown +Date: Fri Sep 7 17:41:34 2018 -0600 + + HLRC: Add Get Lifecycle Policy API to HLRC (#33323) + + Adds Request and Reponse classes for accessing lifecycle policies. + Changes existing tests to use these classes where appropriate. + Sets up SPI configuration to allow parsing *Actions from XContent. + +commit 8fa8dea138feab3a406e8433e70e3910709fad88 +Author: Lee Hinman +Date: Fri Sep 7 16:32:25 2018 -0600 + + Encapsulate Client as class variable for PolicyStepsRegistry (#33529) + + Rather than pass in the client on the `update` step, this makes it passed in to + the constructor so it's not required on every update. + +commit e0b99d7ab36e7ab88f942dad08fdbdfc7a540d26 +Author: Tal Levy +Date: Fri Sep 7 15:04:02 2018 -0700 + + move ILM client objects from protocol to client (#33532) + +commit 609a167c45347c92c945c39882d24e79d8d26941 +Author: Nik Everett +Date: Fri Sep 7 16:45:51 2018 -0400 + + CRUD: Disable wait for refresh tests with delete + + They look to be broken, sadly. + + Tracked by #33533 + +commit 0685c951f2ad090e911f398ad7b3f04d912c8d43 +Author: Nik Everett +Date: Fri Sep 7 16:22:07 2018 -0400 + + Test: Fix test name (#33510) + + This test has the wrong name and hasn't been automatically running. This + fixes the name so we'll run it. + +commit d6b40f4c7b21f8bc665304a14d05560f999564cd +Author: Michael Basnight +Date: Fri Sep 7 13:44:16 2018 -0500 + + HLRC: split ingest request converters (#33435) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the IngestClient + request converters. + +commit 190ea9a6def9082348d983b16420ef02607d4c17 +Author: Nik Everett +Date: Fri Sep 7 14:31:23 2018 -0400 + + Logging: Configure the node name when we have it (#32983) + + Change the logging infrastructure to handle when the node name isn't + available in `elasticsearch.yml`. In that case the node name is not + available until long after logging is configured. The biggest change is + that the node name logging no longer fixed at pattern build time. + Instead it is read from a `SetOnce` on every print. If it is unset it is + printed as `unknown` so we have something that fits in the pattern. + On normal startup we don't log anything until the node name is available + so we never see the `unknown`s. + +commit 8d61457d9c8ff6471009728e8c8206ff6985999b +Author: Michael Basnight +Date: Fri Sep 7 13:30:57 2018 -0500 + + HLRC: split xpack request converters (#33444) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the XPackClient + request converters. + +commit 43592305a1a1abb318a31216284d5a1b3f0838cf +Author: Michael Basnight +Date: Fri Sep 7 13:08:31 2018 -0500 + + HLRC: split watcher request converters (#33442) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the WatcherClient + request converters. + +commit 9d16a7b7f0bcea1d3139671eaa983e7c6ed5d6c5 +Author: Jay Modi +Date: Fri Sep 7 11:51:37 2018 -0600 + + HLRC: add enable and disable user API support (#33481) + + This change adds support for enable and disable user APIs to the high + level rest client. There is a common request base class for both + requests with specific requests that simplify the use of these APIs. + + The response for these APIs is simply an empty object so a new response + class has been created for cases where we expect an empty response to + be returned. + + Finally, the put user documentation has been moved to the proper + location that is not within an x-pack sub directory and the document + tags no longer contain x-pack. + + See #29827 + +commit 944868908cf326a31043e86c7f4a222fbd221dab +Author: lcawl +Date: Fri Sep 7 10:26:44 2018 -0700 + + [DOCS] Fixes formatting error + +commit ab7e6961083fcaa97df77f71eab3d892515910e8 +Author: Nhat Nguyen +Date: Fri Sep 7 12:58:00 2018 -0400 + + TEST: Ensure merge triggered in _source retention test (#33487) + + We invoke force merge twice in the test to verify that recovery sources + are pruned when the global checkpoint advanced. However, if the global + checkpoint equals to the local checkpoint in the first force-merge, the + second force-merge will be a noop because all deleted docs are expunged + in the first merge already. We need to flush a new segment to make merge + happen so we can verify that all recovery sources are pruned. + +commit e42cc5cd8ce50fdbfb890f91ffc15bf147c112cc +Author: David Roberts +Date: Fri Sep 7 17:41:57 2018 +0100 + + [ML] Add a file structure determination endpoint (#33471) + + This endpoint accepts an arbitrary file in the request body and + attempts to determine the structure. If successful it also + proposes mappings that could be used when indexing the file's + contents, and calculates simple statistics for each of the fields + that are useful in the data preparation step prior to configuring + machine learning jobs. + +commit 4d233107f8c8af910cd3dfabf10f7eec7a35aa58 +Author: Benjamin Trent +Date: Fri Sep 7 11:16:58 2018 -0500 + + HLRC: ML Forecast Job (#33506) + + * HLRC: ML Forecast job + +commit 42469a993058bb140ce4a9a1dd9e7c9e66de539d +Author: Michael Basnight +Date: Fri Sep 7 09:46:27 2018 -0500 + + HLRC: split migration request converters (#33436) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the MigrationClient + request converters. + +commit c32e71c61f8f8d71e603b9186dce31fce51fe020 +Author: Michael Basnight +Date: Fri Sep 7 09:25:47 2018 -0500 + + HLRC: split snapshot request converters (#33439) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the SnapshotClient + request converters. + +commit 90c99ea96e2c65dee37419e4f574eb9c37363ca9 +Author: Daniel Mitterdorfer +Date: Fri Sep 7 15:52:47 2018 +0200 + + Make Watcher validation message copy/pasteable + + Watcher validates `action.auto_create_index` upon startup. If a user + specifies a pattern that does not contain watcher indices, it raises an + error message to include a list of three indices. However, the indices + are separated by a comma and a space which is not considered in parsing. + + With this commit we change the error message string so it does not + contain the additional space thus making it more straightforward to copy + it to the configuration file. + + Closes #33369 + Relates #33497 + +commit 79e79697349234e3e07804264bc5f23772e17ac4 +Author: Colin Goodheart-Smithe +Date: Fri Sep 7 13:10:27 2018 +0100 + + Removes redundant test method in SQL tests (#33498) + +commit f83641346ffd41d795361368e33ea88fbc29de59 +Author: Colin Goodheart-Smithe +Date: Fri Sep 7 13:06:51 2018 +0100 + + Adds checks to ensure index metadata exists when we try to use it (#33455) + + * Adds checks to ensure index metadata exists when we try to use it + + * Fixes failing test + +commit 9230a48722533b1ac523d3f88927494b9acec0a0 +Author: Benjamin Trent +Date: Fri Sep 7 07:04:27 2018 -0500 + + HLRC: ML Post Data (#33443) + + * HLRC: ML Post data + +commit c12d23221557e8d76bf456af1b23315f6527405b +Author: Simon Willnauer +Date: Fri Sep 7 14:00:24 2018 +0200 + + Pass Directory instead of DirectoryService to Store (#33466) + + Instead of passing DirectoryService which causes yet another dependency + on Store we can just pass in a Directory since we will just call + `DirectoryService#newDirectory()` on it anyway. + +commit 017ffe5d127ce0fb33feb8fc4e5fededb8cbfda1 +Merge: 21bb4720a26 79cd6385fe6 +Author: Colin Goodheart-Smithe +Date: Fri Sep 7 10:59:10 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 79cd6385fe68e92645ec10c5b05ec75c0277cd4a +Author: Jim Ferenczi +Date: Fri Sep 7 10:58:06 2018 +0200 + + Collapse package structure for metrics aggs (#33463) + + This change collapses all metrics aggregations classes into a single package `org.elasticsearch.aggregations.metrics`. + It also restricts the visibility of some classes (aggregators and factories) that should not be used outside of the package. + + Relates #22868 + +commit 34859414a070332ca97ca51dc98f1a94d2ea527d +Author: Jim Ferenczi +Date: Fri Sep 7 10:30:53 2018 +0200 + + Fix bwc serialization of total hits when track_total_hits is false + +commit 7b923ea60459eab3546db0684b5d53322655cde3 +Author: Costin Leau +Date: Fri Sep 7 07:24:43 2018 +0300 + + SQL: Improve alias vs index resolution (#33393) + + Improve pattern resolution for aliases vs indices + +commit ee73bc2f3fbf8a47d8e68f09dadccb92e5bc2fd9 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Sep 7 08:46:49 2018 +1000 + + [SECURITY] Set Auth-scheme preference (#33156) + + Some browsers (eg. Firefox) behave differently when presented with + multiple auth schemes in 'WWW-Authenticate' header. The expected + behavior is that browser select the most secure auth-scheme before + trying others, but Firefox selects the first presented auth scheme and + tries the next ones sequentially. As the browser interpretation is + something that we do not control, we can at least present the auth + schemes in most to least secure order as the server's preference. + + This commit modifies the code to collect and sort the auth schemes + presented by most to least secure. The priority of the auth schemes is + fixed, the lower number denoting more secure auth-scheme. + The current order of schemes based on the ES supported auth-scheme is + [Negotiate, Bearer,Basic] and when we add future support for + other schemes we will need to update the code. If need be we will make + this configuration customizable in future. + + Unit test to verify the WWW-Authenticate header values are sorted by + server preference as more secure to least secure auth schemes. + Tested with Firefox, Chrome, Internet Explorer 11. + + Closes#32699 + +commit 0d45752e50b7b7dd84a397fb95e133bcd28dd945 +Author: Nik Everett +Date: Thu Sep 6 17:55:24 2018 -0400 + + Fix IndexMetaData loads after rollover (#33394) + + When we rollover and index we write the conditions of the rollover that + the old index met into the old index. Loading this index metadata + requires a working `NamedXContentRegistry` that has been populated with + parsers from the rollover infrastructure. We had a few loads that didn't + use a working `NamedXContentRegistry` and so would fail if they ever + encountered an index that had been rolled over. Here are the locations + of the loads and how I fixed them: + + * IndexFolderUpgrader - removed entirely. It existed to support opening + indices made in Elasticsearch 2.x. Since we only need this change as far + back as 6.4.1 which will supports reading from indices created as far + back as 5.0.0 we should be good here. + * TransportNodesListGatewayStartedShards - wired the + `NamedXContentRegistry` into place. + * TransportNodesListShardStoreMetaData - wired the + `NamedXContentRegistry` into place. + * OldIndexUtils - removed entirely. It existed to support the zip based + index backwards compatibility tests which we've since replaced with code + that actually runs old versions of Elasticsearch. + + In addition to fixing the actual problem I added full cluster restart + integration tests for rollover which would have caught this problem and + I added an extra assertion to IndexMetaData's deserialization code which + will trip if we try to deserialize and index's metadata without a fully + formed `NamedXContentRegistry`. It won't catch if use the *wrong* + `NamedXContentRegistry` but it is better than nothing. + + Closes #33316 + +commit 21bb4720a26e900f4bf5027d06eed462e68f9369 +Author: Tal Levy +Date: Thu Sep 6 13:32:24 2018 -0700 + + add notion of version and modified_date to LifecyclePolicyMetadata (#33450) + + It is useful to keep track of which version of a policy is currently + being executed by a specific index. For management purposes, it would + also be useful to know at which time the latest version was inserted + so that an audit trail is left for reconciling changes happening in ILM. + +commit 6a3adbd93565299c9e2c09b3781a2d051d22550d +Author: Michael Basnight +Date: Thu Sep 6 14:11:27 2018 -0500 + + HLRC: split tasks request converters (#33441) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the TasksClient + request converters. + +commit b335487ca66bc6f1375e8d825fd81200d1ffd47f +Author: Lee Hinman +Date: Thu Sep 6 11:22:27 2018 -0600 + + Fix qa build.gradle to gradle assemble works correctly + + There is a new way to disable assembling from certain subdirectories + +commit cd4bdde328b927762eeb98a19a44377166cb305f +Author: lcawl +Date: Thu Sep 6 09:53:03 2018 -0700 + + [DOCS] Moves troubleshooting and limitations to stack-docs + +commit c6c456e8cbd102a2aee16097c8c07f60f77466e1 +Author: Simon Willnauer +Date: Thu Sep 6 18:48:05 2018 +0200 + + Move up acquireSearcher logic to Engine (#33453) + + By moving the logic to acquire the searcher up to the engine + it's simpler to build new engines that are for instance read-only. + +commit 8ce4ceb59e64e6d238137582dfaa09a7cb9ed07a +Author: Michael Basnight +Date: Thu Sep 6 11:37:27 2018 -0500 + + HLRC: split license request converters (#33438) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the LicenseClient + request converters. + +commit 5251300fb5124b4a47d91302dd6ac2dd58c9ee50 +Author: Michael Basnight +Date: Thu Sep 6 11:33:26 2018 -0500 + + HLRC: split graph request converters (#33429) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the GraphClient + request converters. + +commit 7441c0376e8ac8cf437bec75e41c112e21f4934a +Author: Lisa Cawley +Date: Thu Sep 6 09:20:42 2018 -0700 + + [DOCS] Adds delete forecast API (#33401) + +commit 8afe09a7494bb34dec29f3f3cc4391547afdff62 +Author: Nhat Nguyen +Date: Thu Sep 6 11:59:16 2018 -0400 + + Pass TranslogRecoveryRunner to engine from outside (#33449) + + This commit allows us to use different TranslogRecoveryRunner when + recovering an engine from its local translog. This change is a + prerequisite for the commit-based rollback PR. + + Relates #32867 + +commit 443f9cadddd44249f5e4ed75c8ab0e7b97135931 +Author: Costin Leau +Date: Thu Sep 6 01:19:49 2018 +0300 + + DOC: Enhance SQL Functions documentation + + Split function section into multiple chapters + Add String functions + Add (small) section on Conversion/Cast functions + Add missing aggregation functions + Enable documentation testing (was disabled by accident). While at it, + fix failing tests + Improve spec tests to allow multi-line queries (useful for docs) + Add ability to ignore a spec test (name should end with -Ignore) + +commit 7ad71f906a9608e1c168358624f4b0a39b240296 +Author: Jim Ferenczi +Date: Thu Sep 6 14:42:06 2018 +0200 + + Upgrade to a Lucene 8 snapshot (#33310) + + The main benefit of the upgrade for users is the search optimization for top scored documents when the total hit count is not needed. However this optimization is not activated in this change, there is another issue opened to discuss how it should be integrated smoothly. + Some comments about the change: + * Tests that can produce negative scores have been adapted but we need to forbid them completely: #33309 + + Closes #32899 + +commit 9b6bbc0182229c4db32f4dff9d83bc878c2eb74e +Author: Benjamin Trent +Date: Thu Sep 6 07:18:09 2018 -0500 + + HLRC: ML Update Job (#33392) + + * HLRC: ML Update Job + +commit ef207edbf0ce3edb408b37de733aedc816d778c1 +Author: Martijn van Groningen +Date: Thu Sep 6 14:14:24 2018 +0200 + + test: do not schedule when test has stopped + +commit cdd82bb2034b4b34a3d7d7424b7a14baea232fd1 +Author: Martijn van Groningen +Date: Thu Sep 6 11:48:51 2018 +0200 + + test: fetch `SeqNoStats` inside try-catch block + + Relates to #33457 + +commit 0849b98f60f64439f8298192a9b9c8444e6d7b66 +Author: David Roberts +Date: Thu Sep 6 09:13:08 2018 +0100 + + [ML] Rename log structure to file structure (#33421) + + Many files supplied to the upcoming ML data preparation + functionality will not be "log" files. For example, + CSV files are generally not "log" files. Therefore it + makes sense to rename library that determines the + structure of these files. + + Although "file structure" could be considered too broad, + as the library currently only works with a few text + formats, in the future it may be extended to work with + more formats. + +commit e134f9b5f3dd93b422a738bf9f08b6672bf66d7a +Author: Alan Woodward +Date: Thu Sep 6 09:04:22 2018 +0100 + + Fix generics in ScriptPlugin#getContexts() (#33426) + + Changes the return value from List to List> to remove raw-types warnings. + +commit 82fab40099c640952c09719330f6511a1a7cd28e +Author: Alexander Reelsen +Date: Thu Sep 6 09:56:20 2018 +0200 + + Core: Fix IndicesSegmentResponse.toXcontent() serialization (#33414) + + When index sorting is enabled, toXContent tried to serialize an + SortField object, resulting in an exception, when using the _segments + endpoint. + + Relates #29120 + +commit 5236f2b1af88bd493c05ea619b96877d38e940bf +Author: Daniel Mitterdorfer +Date: Thu Sep 6 09:42:45 2018 +0200 + + Improve reproducability of RestControllerTests + + With this commit we use the classic parent circuit breaker which does + not account for real memory usage. In those tests we want to have + reproducible results and hence it makes sense to disable the real memory + circuit breaker there. + +commit b1257d873bc8541b15d7c933a383344c151a1333 +Merge: 96d515e3f58 a721d09c81f +Author: Colin Goodheart-Smithe +Date: Thu Sep 6 08:17:40 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit a721d09c81fc0881f3796550d6c35d459418b883 +Author: Martijn van Groningen +Date: Thu Sep 6 08:01:58 2018 +0200 + + [CCR] Added auto follow patterns feature (#33118) + + Auto Following Patterns is a cross cluster replication feature that + keeps track whether in the leader cluster indices are being created with + names that match with a specific pattern and if so automatically let + the follower cluster follow these newly created indices. + + This change adds an `AutoFollowCoordinator` component that is only active + on the elected master node. Periodically this component checks the + the cluster state of remote clusters if there new leader indices that + match with configured auto follow patterns that have been defined in + `AutoFollowMetadata` custom metadata. + + This change also adds two new APIs to manage auto follow patterns. A put + auto follow pattern api: + + ``` + PUT /_ccr/_autofollow/{{remote_cluster}} + { + "leader_index_pattern": ["logs-*", ...], + "follow_index_pattern": "{{leader_index}}-copy", + "max_concurrent_read_batches": 2 + ... // other optional parameters + } + ``` + + and delete auto follow pattern api: + + ``` + DELETE /_ccr/_autofollow/{{remote_cluster_alias}} + ``` + + The auto follow patterns are directly tied to the remote cluster aliases + configured in the follow cluster. + + Relates to #33007 + + + Co-authored-by: Jason Tedor jason@tedor.me + +commit d71ced1b0077c41d84ad3842f7aa98897bcb450e +Author: Jason Tedor +Date: Wed Sep 5 20:43:44 2018 -0400 + + Generalize search.remote settings to cluster.remote (#33413) + + With features like CCR building on the CCS infrastructure, the settings + prefix search.remote makes less sense as the namespace for these remote + cluster settings than does a more general namespace like + cluster.remote. This commit replaces these settings with cluster.remote + with a fallback to the deprecated settings search.remote. + +commit 39e3bd93c7f5554720ea7a6820d20082c7feeeb0 +Author: Nhat Nguyen +Date: Wed Sep 5 19:05:41 2018 -0400 + + TEST: Create following engines in the main thread (#33391) + + There are two races in the testUpdateAndReadChangesConcurrently if the + following engines are created in the worker threads. We fixed the + translog issue in #33352, but there is still another race with + createStore. + + This commit ensures that we create all engines in the main thread. + + Relates #33352 + Closes #33344 + +commit 41839cf9a857cb1eef743f36b51a55d10f7073e1 +Author: Nhat Nguyen +Date: Wed Sep 5 19:03:34 2018 -0400 + + Acquire seacher on closing engine should throw ACE (#33331) + + Closes #33330 + +commit 96d515e3f58c8f764d24feecdd3528c271e4e953 +Author: Lee Hinman +Date: Wed Sep 5 16:37:45 2018 -0600 + + Replace PhaseAfterStep with PhaseCompleteStep (#33398) + + This removes `PhaseAfterStep` in favor of a new `PhaseCompleteStep`. This step + in only a marker that the `LifecyclePolicyRunner` needs to halt until the time + indicated for entering the next phase. + + This also fixes a bug where phase times were encapsulated into the policy + instead of dynamically adjusting to policy changes. + + Supersedes #33140, which it replaces + Relates to #29823 + +commit b697f485bb4815b231f4accb5725fdc237214aef +Author: Tim Brooks +Date: Wed Sep 5 16:12:37 2018 -0600 + + Introduce `TransportLogger` for common logging (#32725) + + Historically we have had a ESLoggingHandler in the netty module that + logs low-level connection operations. This class just extends the netty + logging handler with some (broken) message deserialization. This commit + fixes this message serialization and moves the class to server. + + This new logger logs inbound and outbound messages. Eventually, we + should move other event logging to this class (connect, close, flush). + That way we will have consistent logging regards of which transport is + loaded. + + Resolves #27306 on master. Older branches will need a different fix. + +commit 88c178dca6587c6f7f0f7b78951284553df0d8b3 +Author: Tim Brooks +Date: Wed Sep 5 16:12:10 2018 -0600 + + Add sni name to SSLEngine in netty transport (#33144) + + This commit is related to #32517. It allows an "server_name" + attribute on a DiscoveryNode to be propagated to the server using + the TLS SNI extentsion. This functionality is only implemented for + the netty security transport. + +commit 6f9c9ab5e1199f6f0823cb2cec6a1ef6dff3f225 +Author: Jake Landis +Date: Wed Sep 5 14:04:25 2018 -0700 + + [ingest] geo-ip performance improvements (#33029) + + Re-implement the cache to avoid jackson JSON de-serialization for + every IP lookup. The built in maxmind cache caches JsonNode objects. + This requires de-serialization for every lookup, even if the object + is found in cache. Profiling shows that is very expensive (CPU). + + The cache will now consist of the fully de-serialized objects. + Profiling shows that the new footprint for the CityDB is ~6KB per cache + entry. This may result in ~6MB increase with the 1000 entry default. + The performance has been measured up to 40% faster on a modern 4 core/8 thread + CPU for an ingest (minimal indexing) workflow. + + Further, the since prior implementation cached the JsonNode objects, + and there is not a 1:1 relationship between an IP lookup / JsonNode + object, the default cache size was most likely too small to be very + effective. While this change does not change the 1000 default cache + size, it will now cache more since there is now a 1:1 relationship between + an IP lookup and value in the cache. + +commit 8b8ff2bc6e2a0a4a20c202cce67c165d09112101 +Author: Jake Landis +Date: Wed Sep 5 14:03:21 2018 -0700 + + HLRC: Add ILM Status to HLRC (#33283) + + Adds support for the Index Lifecycle Management Status to the Java + High-Level Rest Client. + + Relates to #33100 + +commit 023e1bf889e1d95562f256522003274c9b327905 +Author: Tal Levy +Date: Wed Sep 5 13:21:50 2018 -0700 + + fix test + +commit ef1066d7f876d63869cfbd3ed73c3c383f8aa69c +Author: Armin Braun +Date: Wed Sep 5 22:04:53 2018 +0200 + + INGEST: Allow Repeated Invocation of Pipeline (#33419) + + * Allows repeated, non-recursive invocation + of the same pipeline + +commit b5f7fb688251d198525a0dd75852a2e91aee46fb +Merge: 0f8bc10bcf1 50e07dd413e +Author: Tal Levy +Date: Wed Sep 5 12:56:58 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 0f8bc10bcf1e3df873e9fd582e3b0f5a1d939395 +Author: Tal Levy +Date: Wed Sep 5 11:35:20 2018 -0700 + + add new phase definition setting used for retrieving phase to execute (#33289) + + Since policies can be updated independent of execution plans for the current + phase being executed, it would be nice to know what the phase that is executing + looks like in JSON. This PR does just that, while also using that index setting + to recontruct the phase steps to execute (for consistency) + +commit 50e07dd413ea1ca009d15c5ecc77f88c83200f6b +Author: Jim Ferenczi +Date: Wed Sep 5 19:57:36 2018 +0200 + + Add an index setting to control TieredMergePolicy#deletesPctAllowed (#32907) + + This change adds an expert index setting called `index.merge.policy.deletes_pct_allowed`. + It controls the maximum percentage of deleted documents that is tolerated in the index. + Lower values make the index more space efficient at the expense of increased CPU and I/O activity. + Values must be between `20` and `50`. Default value is `33`. + +commit 5c624bc55b2cc26ec8986e4bd69c8c48c06db7f4 +Author: Nik Everett +Date: Wed Sep 5 13:04:26 2018 -0400 + + Logging: Further clean up logging ctors (#33378) + + Drops and unused logging constructor, simplifies a rarely used one, and + removes `Settings` from a third. There is now only a single logging ctor + that takes `Settings` and we'll remove that one in a follow up change. + +commit 46ac8d1a51339972f9647775155dc1a2ee1ca624 +Author: Adrien Grand +Date: Wed Sep 5 18:58:22 2018 +0200 + + Make test less GC-intensive. + +commit ea52277a1ecb044c4753f900bd785e035a01d100 +Author: Jay Modi +Date: Wed Sep 5 10:56:30 2018 -0600 + + HLRest: add put user API (#32332) + + This commit adds a security client to the high level rest client, which + includes an implementation for the put user api. As part of these + changes, a new request and response class have been added that are + specific to the high level rest client. One change here is that the response + was previously wrapped inside a user object. The plan is to remove this + wrapping and this PR adds an unwrapped response outside of the user + object so we can remove the user object later on. + + See #29827 + +commit 7eef7f441b502ab522db2aa18fbe6de87289f644 +Author: Adrien Grand +Date: Wed Sep 5 18:25:32 2018 +0200 + + Add info about the running threads in case of failure. + + Relates #29880 + +commit eafc2a5470fc8000e708fde430454139c7de91ce +Author: Christoph Büscher +Date: Wed Sep 5 18:27:21 2018 +0200 + + Don't count metadata fields towards index.mapping.total_fields.limit (#33386) + + The maximum number of fields per index is limited to 1000 by default by the + `index.mapping.total_fields.limit` setting to prevent accidental mapping + explosions due to too many fields. Currently all metadata fields also count + towards this limit, which can lead to some confusion when using lower limits. + It is not obvious for users that they cannot actually add as many fields as + are specified by the limit in this case. + + This change takes the number of metadata fields out of the field count that we + check against the field limit. It also adds tests that check that we can add + fields up to the specified limit, but throw an exception for any additional field added. + + Closes #24096 + +commit 23934e39d2911c987bd7c91cbf5e8281ced361a9 +Author: Jason Tedor +Date: Wed Sep 5 11:01:58 2018 -0400 + + Fix deprecated setting specializations (#33412) + + Deprecating a some setting specializations (e.g., list settings) does + not cause deprecation warning headers and deprecation log messages to + appear. This is due to a missed check for deprecation. This commit fixes + this for all setting specializations, and ensures that this can not be + missed again. + +commit 7319bc7411fb289f854ebbbdb4ac0e91235b3e34 +Author: Michael Basnight +Date: Wed Sep 5 09:34:47 2018 -0500 + + HLRC: split cluster request converters (#33400) + + In an effort to encapsulate the different clients, the request + converters are being shuffled around. This splits the ClusterClient + request converters. + +commit 24776b2b8017f083be8d848bfa2f33bb2014d4db +Author: Dimitris Athanasiou +Date: Wed Sep 5 15:06:55 2018 +0100 + + HLRC: Add ML get influencers API (#33389) + + Relates #29827 + +commit 636442700c02bdf929cad6dd47f18a84635309c4 +Author: Alan Woodward +Date: Wed Sep 5 14:52:43 2018 +0100 + + Add conditional token filter to elasticsearch (#31958) + + This allows tokenfilters to be applied selectively, depending on the status of the current token in the tokenstream. The filter takes a scripted predicate, and only applies its subfilter when the predicate returns true. + +commit 74b87989d976395b333bb478bd99e760b0878344 +Author: Nik Everett +Date: Wed Sep 5 09:17:02 2018 -0400 + + Build: Merge xpack checkstyle config into core (#33399) + + Solves all of the xpack line length suppressions and then merges the + remainder of the xpack checkstyle_suppressions.xml file into the core + checkstyle_suppressions.xml file. At this point that just means the + antlr generated files for sql. + + It also adds an exclusion to the line length tests for javadocs that + are just a URL. We have one such javadoc and breaking up the line would + make the link difficult to use. + +commit 913d5fd820477c9277c6c34a757cab258bbe9041 +Author: Adrien Grand +Date: Wed Sep 5 14:52:47 2018 +0200 + + Disable IndexRecoveryIT.testRerouteRecovery. + + Relates #32686. + +commit 46774098d9fcdbf7b983c05b6afd76b425be7f60 +Author: Armin Braun +Date: Wed Sep 5 14:25:29 2018 +0200 + + INGEST: Implement Drop Processor (#32278) + + * INGEST: Implement Drop Processor + * Adjust Processor API + * Implement Drop Processor + * Closes #23726 + +commit a2968292053a8b3497c1ce384166b8bd33afe12f +Author: David Roberts +Date: Wed Sep 5 12:57:20 2018 +0100 + + [ML] Add field stats to log structure finder (#33351) + + The log structure endpoint will return these in addition to + pure structure information so that it can be used to drive + pre-import data visualizer functionality. + + The statistics for every field are count, cardinality + (distinct count) and top hits (most common values). Extra + statistics are calculated if the field is numeric: min, max, + mean and median. + +commit a7b5f2c07d18d6943b1ebc6e8e7bdd8675c978eb +Author: Colin Goodheart-Smithe +Date: Wed Sep 5 12:49:56 2018 +0100 + + Adds ILMClient for use with transport client (#33357) + + * Adds ILMClient for use with transport client + + * add sync methods + + * adds ILMClient to XPackClient + +commit c303006e6b79e96e643473e5bc77e901e4972ffc +Author: Paul Sanwald +Date: Wed Sep 5 07:35:59 2018 -0400 + + Add interval response parameter to AutoDateInterval histogram (#33254) + + Adds the interval used to the aggregation response. + +commit 4156cc3faefe3a1c0b0af589de349beb4ad4855f +Author: Armin Braun +Date: Wed Sep 5 12:08:28 2018 +0200 + + MINOR+CORE: Remove Dead Methods ClusterService (#33346) + + * None of these methods are used anywhere + +commit f00a28a909be71e17f0d1bc4c4a6ee15c5fdd0cb +Merge: 30544cbe7ad d7965ba681a +Author: Colin Goodheart-Smithe +Date: Wed Sep 5 09:48:48 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit d7965ba681a92143325d234f6158ad559e15fd1c +Author: Costin Leau +Date: Wed Sep 5 10:44:18 2018 +0300 + + SQL: Align SYS TABLE for ODBC SQL_ALL_* args (#33364) + + Fix a bug in SYS TABLES command that did skipped SQL_ALL_* arguments for + catalog and table types + + Fix #33312 + +commit 9f96d2ce1760c4d5814e1009af53e9046d0dad65 +Author: Alpar Torok +Date: Wed Sep 5 08:24:44 2018 +0300 + + Don't build bwc on assemble (#33372) + + Gradle triggers the build of artifacts even if assemble is disabled. + Most users will not need bwc distributions after running `./gradlew + assemble` so instead of forcing them to add `-x buildBwcVersion`, we + detect this and skip the configuration of the artifacts. + +commit cfd3fa72ed9594f51b1580da43f1d6c49de00afa +Author: Gordon Brown +Date: Tue Sep 4 16:14:18 2018 -0600 + + Add user-defined cluster metadata (#33325) + + Adds a place for users to store cluster-wide data they wish to associate + with the cluster via the Cluster Settings API. This is strictly for + user-defined data, Elasticsearch makes no other other use of these + settings. + +commit 48790b0aad0f8cc12da288f1bc4a5bd6a3566d38 +Author: Jason Tedor +Date: Tue Sep 4 17:50:31 2018 -0400 + + Allow query caching by default again (#33328) + + With the introduction of the default distribution, it means that by + default the query cache is wrapped in the security implementation of the + query cache. This cache does not allow caching if the request does not + carry indices permissions. Yet, this will not happen if authorization is + not allowed, which it is not by default. This means that with the + introduction of the default distribution, query caching was disabled by + default! This commit addresses this by checking if authorization is + allowed and if not, delegating to the default indices query + cache. Otherwise, we proceed as before with security. Additionally, we + clear the cache on license state changes. + +commit 30544cbe7ada99c19a3ae17c4005333cf66d7318 +Author: Colin Goodheart-Smithe +Date: Tue Sep 4 18:03:07 2018 +0100 + + Fixes Rendering of after in Phase (#33304) + + Closes #33282 + +commit f3f8d9b8337d7e7ea378cba01c95d647e39715e6 +Author: Lisa Cawley +Date: Tue Sep 4 10:02:13 2018 -0700 + + [DOCS] Moves monitoring pages to docs folder (#33324) + +commit bd8d82d85972faa4bdd2685f8a9f8c7afff4445f +Author: Dimitris Athanasiou +Date: Tue Sep 4 17:31:49 2018 +0100 + + [ML] Add missing param description in GetOverallBucketsRequest + +commit c5109a54ee77afd23b9fa94b4d1f52c1fa648580 +Author: lcawl +Date: Tue Sep 4 09:26:28 2018 -0700 + + [DOCS] Revert fix for broken link + +commit 43f80fa82b8a69763665bb45744703bd063e2d19 +Author: Costin Leau +Date: Tue Sep 4 19:05:21 2018 +0300 + + DOCS: Fix anchor and example typos + +commit 303ae25a6a0922b44be6a6b10b98db9782a91351 +Author: lcawl +Date: Tue Sep 4 09:05:30 2018 -0700 + + [DOCS] Fixes broken link + +commit ebd5eb6dc2cfeb5a591de8cb0685ea08bb7b5b3e +Author: Nik Everett +Date: Tue Sep 4 11:10:44 2018 -0400 + + ML: Fix build after HLRC change + + I recently merged a HLRC change that passed the PR builds but didn't + compile after merging. Sad time. This fixes the compilation. + +commit dbc7102c86d15cce29221c377fdc60570427de99 +Author: Jim Ferenczi +Date: Tue Sep 4 16:25:52 2018 +0200 + + Fix inner hits retrieval when stored fields are disabled (_none_) (#33018) + + Now that types are unique per mapping we can retrieve the document mapper + without referencing the type. This fixes an NPE when stored fields are disabled. + For 6x we'll need a different fix since mappings can still have multiple types. + + Relates #32941 + +commit 17c7f99343e59e02f95a85c3d8c130f242ed9867 +Author: Costin Leau +Date: Tue Sep 4 16:54:10 2018 +0300 + + SQL: Show/desc commands now support table ids (#33363) + + Extend SHOW TABLES, DESCRIBE and SHOW COLUMNS to support table + identifiers not just SQL LIKE pattern. + This allows both Elasticsearch-style multi-index patterns and SQL LIKE. + To disambiguate between the two (as the " vs ' can be easy to miss), + the grammar now requires LIKE keyword as a prefix for all LIKE-like + patterns. + + Also added some docs comparing the two types of patterns. + + Fix #33294 + +commit 16b53b5ab548f8d97ec598a682f46a8fa5d7882d +Author: Nhat Nguyen +Date: Tue Sep 4 09:03:26 2018 -0400 + + Mute testValidateFollowingIndexSettings + + Tracked at #33379 + +commit 761e8c461f8687e0c79f1ec4f225ad7de560ae56 +Author: Sohaib Iftikhar +Date: Tue Sep 4 14:56:26 2018 +0200 + + HLRC: Add delete by query API (#32782) + + Adds the delete-by-query API to the High Level REST Client. + +commit 1457b07a067cfaacffac7235091a8c371b0a7895 +Author: Dimitris Athanasiou +Date: Tue Sep 4 11:38:24 2018 +0100 + + [ML] The sort field on get records should default to the record_score (#33358) + + This is not changing the behaviour as when the sort field was set + to `influencer_score` the secondary sort would be used and that + was using the `record_score` at the highest priority. + +commit 92ab442aee06dc0304ff4bff4835c7032720d069 +Merge: 201e1390136 0bf36253a9d +Author: Colin Goodheart-Smithe +Date: Tue Sep 4 10:34:49 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 84eaac79d7657893bd7fd1f6fc69d4260cf0fa56 +Author: David Roberts +Date: Tue Sep 4 09:43:49 2018 +0100 + + [ML] Minor improvements to categorization Grok pattern creation (#33353) + + 1. The TOMCAT_DATESTAMP format needs to be checked before + TIMESTAMP_ISO8601, otherwise TIMESTAMP_ISO8601 will + match the start of the Tomcat datestamp. + 2. Exclude more characters before and after numbers. For + example, in 1.2.3 we don't want to match 1.2 as a float. + +commit d9f394b099b86835aa3e42f28dc42063c37f72ed +Author: Nikolay Vasiliev +Date: Tue Sep 4 10:07:11 2018 +0200 + + [DOCS] fix a couple of typos (#33356) + +commit 7f7e8fd733d3f98e7422288099b530610b4c4d1e +Author: Alpar Torok +Date: Tue Sep 4 07:32:14 2018 +0300 + + Disable assemble task instead of removing it (#33348) + +commit 78df00ff245efb16e1a992573cc49d5708c90901 +Author: Julie Tibshirani +Date: Tue Sep 4 01:15:19 2018 +0000 + + Simplify the return type of FieldMapper#parse. (#32654) + +commit 767d8e0801c2e8cfe8ca655262982c7826fa4b7c +Author: Benjamin Trent +Date: Mon Sep 3 19:06:18 2018 -0500 + + [ML] Delete forecast API (#31134) (#33218) + + * Delete forecast API (#31134) + +commit 09bf4e5f007492b5f765941080518c2de2381b35 +Author: Jason Tedor +Date: Mon Sep 3 19:17:57 2018 -0400 + + Introduce private settings (#33327) + + This commit introduces the formal notion of a private setting. This + enables us to register some settings that we had previously not + registered as fully-fledged settings to avoid them being exposed via + APIs such as the create index API. For example, we had hacks in the + codebase to allow index.version.created to be passed around inside of + settings objects, but was not registered as a setting so that if a user + tried to use the setting on any API then they would get an + exception. This prevented users from setting index.version.created on + index creation, or updating it via the index settings API. By + introducing private settings, we can continue to reject these attempts, + yet now we can represent these settings as actual settings. In this + change, we register index.version.created as an actual setting. We do + not cutover all settings that we had been treating as private in this + pull request, it is already quite large due to moving some tests around + to account for the fact that some tests need to be able to set the + index.version.created. This can be done in a follow-up change. + +commit 79db16f9bb1a204fc19b8ec3f4fddd5aaeac6da1 +Author: Christoph Büscher +Date: Mon Sep 3 20:56:05 2018 +0200 + + [Docs] Add search timeout caveats (#33354) + + Global search timeouts and timeouts specified in the search request body use the + same internal mechanism as search cancellation. Therefore the same caveats + apply, mostly around the responsiveness of the timeout which gets only checked + by a running search on segment boundaries by default. + + Closes #31263 + +commit 1f046617bf8891e194653bd2b46dda0c16a95bc7 +Author: Armin Braun +Date: Mon Sep 3 19:55:59 2018 +0200 + + TESTS: Fix Race Condition in Temp Path Creation (#33352) + + * TESTS: Fix Race Condition in Temp Path Creation + + * Calling `createTempDir` concurrently here in + the `Follower`s causes collisions at times + which lead to `createEngine` throwing because + of unexpected files in the newly created temp + dir + * Fixed by creating all temp dirs in the main test thread + * closes #33344 + +commit 24d60c7f4b5d46d0f44c4c03b9b15b9faa6ba2df +Author: Nhat Nguyen +Date: Mon Sep 3 11:58:49 2018 -0400 + + Fix from_range in search_after in changes snapshot (#33335) + + We can have multiple documents in Lucene with the same seq_no for + parent-child documents (or without rollback). In this case, the usage + "lastSeenSeqNo + 1" is an off-by-one error as it may miss some + documents. This error merely affects the `skippedOperations` contract. + + See: https://github.com/elastic/elasticsearch/pull/33222#discussion_r213842257 + + Closes #33318 + +commit 42424aff21e1832f85a91769cc2cf1317f2a21e9 +Author: Armin Braun +Date: Mon Sep 3 17:06:12 2018 +0200 + + TESTS+DISTR.: Fix testIndexCheckOnStartup Flake (#33349) + + * Ignore all `RuntimeException` since random + file corruption triggers other RTE in addition + to the randomly caught one + * closes #33345 + +commit a9d2b1dde82d0400b47a9c7208f71e2d2e8133ee +Author: tony-dillon +Date: Mon Sep 3 15:49:53 2018 +0100 + + Null completion field should not throw IAE (#33268) + + Ignore null value on the completion field + + Closes #33200 + +commit 0bf36253a9db335775475ace7bb9964689b48b3c +Author: Colin Goodheart-Smithe +Date: Mon Sep 3 14:54:17 2018 +0100 + + Adds code to help with IndicesRequestCacheIT failures (#33313) + + * Adds code to help with IndicesRequestCacheIT failures + + Relates to #32827 + + * Adds comment + + * Fixes test failure + +commit 201e1390136148aa23387c388534a6677860c8a4 +Author: Colin Goodheart-Smithe +Date: Mon Sep 3 14:51:43 2018 +0100 + + Adds documentation about defining rollover action in ILM (#33243) + + * Adds documentation about defining rollover aciton in ILM + + * Applies review comments + + * Addresses further review comments + + * Fixes Docs test + +commit ccb2ad25cc2c5a739c39c8ecf82d064c9baa506a +Author: David Kyle +Date: Mon Sep 3 13:35:04 2018 +0100 + + Prevent NPE parsing the stop datafeed request. (#33347) + + The issue depends on the request parameters being passed in the request + body rather than as query parameters. + +commit 54fe7fb5a1fc118b251c5aed8ce27c8d9885d2c8 +Author: Dimitris Athanasiou +Date: Mon Sep 3 13:34:36 2018 +0100 + + HLRC: Add ML get overall buckets API (#33297) + + Relates #29827 + +commit 246a7df8c23233747c7b8b8b00b134bcbf6919bc +Author: Alexander Reelsen +Date: Mon Sep 3 13:13:19 2018 +0200 + + Core: Fix epoch millis java time formatter (#33302) + + The existing implemention could not deal with negative numbers as well + as +- 999 milliseconds around the epoch. + + This commit uses Instant.ofEpochMilli() and parses the input to + a number instead of using a date formatter. + +commit 978d1ed2575d622cb5055f6fa7fbe845ce58572c +Author: Christoph Büscher +Date: Mon Sep 3 11:09:30 2018 +0200 + + [Docs] Improve tuning for speed advice (#33315) + + This change merges two sections in the "Tune for search speed" documentation + that recommend mapping numeric identifiers as keywords. Both sections contain + mostly the same advice, so they can be merged. + + Closes #32733 + +commit e2c1beb1be5b47d124903883bc31be8a648c8c03 +Merge: 3eef74d5d58 9310d2eaf32 +Author: Colin Goodheart-Smithe +Date: Mon Sep 3 10:01:16 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 90ce3a6224121fce37debfe11e3530bc059997e1 +Author: Zachary Tong +Date: Mon Sep 3 04:49:19 2018 -0400 + + [Rollup] Fix Caps Comparator to handle calendar/fixed time (#33336) + + The comparator used TimeValue parsing, which meant it couldn't handle + calendar time. This fixes the comparator to handle either (and potentially + mixed). The mixing shouldn't be an issue since the validation code + upstream will prevent it, but was simplest to allow the comparator + to handle both. + +commit 9310d2eaf32d91cc83a71cd0979c7a2e12e66071 +Author: Jim Ferenczi +Date: Mon Sep 3 10:27:42 2018 +0200 + + [CI] Mute IndexShardTests#testIndexCheckOnStartup fails #33345 + +commit 2fa75b4438fe33d4e39d5e932e0231f76a4c7514 +Author: Jim Ferenczi +Date: Mon Sep 3 10:14:00 2018 +0200 + + [CI] Mute LuceneChangesSnapshotTests#testUpdateAndReadChangesConcurrently + +commit f0a61b6decc8ba6e662f410389bb45665b1cd840 +Author: Jim Ferenczi +Date: Mon Sep 3 09:36:39 2018 +0200 + + Security for _field_names field should not override field statistics (#33261) + + In Lucene 8 the statistics for a field (doc_count, sum_doc_count, ...) are + checked and invalid values (v < 0) are rejected. Though for the _field_names + field we hide the statistics of the field if security is enabled since + some terms (field names) may be filtered. However this statistics are never + used, this field is not used for ranking and cannot be used to generate + term vectors. For these reasons this commit restores the original statistics + for the field in order to be compliant with Lucene 8. + +commit 713c07e14d2f5b854f578ea70b8da6e44841740c +Author: Jim Ferenczi +Date: Mon Sep 3 09:34:35 2018 +0200 + + Add early termination support to BucketCollector (#33279) + + This commit adds the support to early terminate the collection of a leaf + in the aggregation framework. This change introduces a MultiBucketCollector which + handles CollectionTerminatedException exactly like the Lucene MultiCollector. + Any aggregator can now throw a CollectionTerminatedException without stopping + the collection of a sibling aggregator. This is useful for aggregators that + can infer their result without visiting all documents (e.g.: a min/max aggregation on a match_all query). + +commit 3c367a2c463e6bc0d0de83e5c84c3a1dd542b6cf +Author: Alpar Torok +Date: Mon Sep 3 10:18:34 2018 +0300 + + Fix extractjar task ci (#33272) + + Remove tasks to check license and notice and add build integration test instead. + + Closes #33201 + +commit 3a1dad1050c0c63711ac678564ca03adfd4ba2e3 +Author: Nhat Nguyen +Date: Sun Sep 2 19:17:51 2018 -0400 + + Mute testFollowIndexAndCloseNode + + Tracked at #33337 + +commit f8b7a4dbc8c8f8943d4ab95a71723ec4d1dd460c +Author: Nik Everett +Date: Sun Sep 2 16:51:26 2018 -0400 + + Logging: Drop Settings from some logging ctors (#33332) + + Drops `Settings` from some logging ctors now that they are no longer + needed. This should allow us to stop passing `Settings` around to quite + as many places. + +commit ea4eef86416fe05a443a2b965614173c7847629c +Merge: 3197a6bbdd0 389bf67275f +Author: Jason Tedor +Date: Sun Sep 2 16:07:50 2018 -0400 + + Merge branch 'master' into ccr + + * master: + HLREST: add update by query API (#32760) + +commit 389bf67275fea488b482149093c2efaab8239a4a +Author: Sohaib Iftikhar +Date: Sun Sep 2 21:15:00 2018 +0200 + + HLREST: add update by query API (#32760) + + Adds update by query to the high level rest client. + +commit 3197a6bbdd018163c53831fbc66a2bfde31449e5 +Merge: c6b011f8ea6 6770a456b8a +Author: Nhat Nguyen +Date: Sun Sep 2 09:30:51 2018 -0400 + + Merge branch 'master' into ccr + + * master: + HLRC: ML Flush job (#33187) + HLRC: Adding ML Job stats (#33183) + LLREST: Drop deprecated methods (#33223) + Mute testSyncerOnClosingShard + [DOCS] Moves machine learning APIs to docs folder (#31118) + +commit c6b011f8ea6dde55f361df0e0f52a1aca0f517bf +Author: Nhat Nguyen +Date: Sun Sep 2 09:28:47 2018 -0400 + + TEST: Increase timeout testFollowIndexAndCloseNode (#33333) + + This test fails several times due to timeout when asserting the number + of docs on the following and leading indices. This change reduces + the number of docs to index and increases the timeout. + +commit 6770a456b8a1b054948744b2e661cf6ff55c9e01 +Author: Benjamin Trent +Date: Sat Sep 1 16:01:23 2018 -0500 + + HLRC: ML Flush job (#33187) + + * HLRC: ML Flush job + + * Fixing package, paths, and test + + * Addressing comments + +commit 19b14fa5edde2e0a56ea179b5aeb51c59f852f61 +Author: Benjamin Trent +Date: Sat Sep 1 13:32:18 2018 -0500 + + HLRC: Adding ML Job stats (#33183) + + * HLRC: Adding pojos for get job stats + + HLRC: Adding pojos for job stats request + + * HLRC: Adding job stats pojos + + * HLRC: ML job stats + + * Minor syntax changes and adding license headers + + * minor comment change + + * Moving to client package, minor changes + + * Addressing PR comments + + * removing bad sleep + + * addressing minor comment around test methods + + * adding toplevel random fields for tests + + * addressing minor review comments + +commit f28cddf9518fb83dd9db2ed88a3362dac6c8aeb3 +Author: Nik Everett +Date: Sat Sep 1 11:11:25 2018 -0400 + + LLREST: Drop deprecated methods (#33223) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. In a + long series of PRs I've changed all of the old style requests. This + drops the deprecated methods and will be released with 7.0. + +commit ce635f5f150d1a9b8b9e380f52aac129e79fb381 +Author: Nhat Nguyen +Date: Sat Sep 1 09:53:23 2018 -0400 + + Mute testSyncerOnClosingShard + + Tracked at #33330 + +commit b7a63f7e7dcf24771d0f0a225d3f01d9f969367b +Author: Lisa Cawley +Date: Fri Aug 31 16:49:24 2018 -0700 + + [DOCS] Moves machine learning APIs to docs folder (#31118) + +commit b93507608a5f975f28a4c4639b2a822737bc1e84 +Merge: 66b164c2a61 ca94d052b85 +Author: Nhat Nguyen +Date: Fri Aug 31 19:03:04 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Mute test watcher usage stats output + [Rollup] Fix FullClusterRestart test + Adjust soft-deletes version after backport into 6.5 + completely drop `index.shard.check_on_startup: fix` for 7.0 (#33194) + Fix AwaitsFix issue number + Mute SmokeTestWatcherWithSecurityIT testsi + drop `index.shard.check_on_startup: fix` (#32279) + tracked at + [DOCS] Moves ml folder from x-pack/docs to docs (#33248) + [DOCS] Move rollup APIs to docs (#31450) + [DOCS] Rename X-Pack Commands section (#33005) + TEST: Disable soft-deletes in ParentChildTestCase + Fixes SecurityIntegTestCase so it always adds at least one alias (#33296) + Fix pom for build-tools (#33300) + Lazy evaluate java9home (#33301) + SQL: test coverage for JdbcResultSet (#32813) + Work around to be able to generate eclipse projects (#33295) + Highlight that index_phrases only works if no slop is used (#33303) + Different handling for security specific errors in the CLI. Fix for https://github.com/elastic/elasticsearch/issues/33230 (#33255) + [ML] Refactor delimited file structure detection (#33233) + SQL: Support multi-index format as table identifier (#33278) + MINOR: Remove Dead Code from PathTrie (#33280) + Enable forbiddenapis server java9 (#33245) + +commit ca94d052b85aa03c4ac610b8d3423aaf9598e0f0 +Author: Nhat Nguyen +Date: Fri Aug 31 18:48:19 2018 -0400 + + Mute test watcher usage stats output + + Tracked at #33326 + +commit ebed8f26187fc315b095106a39edbe46d0ed5778 +Author: Zachary Tong +Date: Fri Aug 31 17:18:22 2018 -0400 + + [Rollup] Fix FullClusterRestart test + + We need to wait for the job to fully initialize and start before + we can attempt to stop it. If we don't, it's possible for the stop + API to be called before the persistent task is fully loaded and it'll + throw an exception. + + Closes #32773 + +commit 08b9247ce226bfc47ebe4f859d97b5ed87940e69 +Author: Nhat Nguyen +Date: Fri Aug 31 16:50:08 2018 -0400 + + Adjust soft-deletes version after backport into 6.5 + + Relates #33222 + +commit 00b272af327b7bfff63b5b8305a040b8b6dbfdb2 +Author: Vladimir Dolzhenko +Date: Fri Aug 31 22:05:40 2018 +0200 + + completely drop `index.shard.check_on_startup: fix` for 7.0 (#33194) + + Relates to #32279 + +commit 6a77cb4211498006b681f46b20feb6b957f8e87d +Author: Tim Brooks +Date: Fri Aug 31 13:37:22 2018 -0600 + + Fix AwaitsFix issue number + + In the previous commit where SmokeTestWatcherWithSecurityIT tests were + muted, I added the incorrect issue numbers. This commit fixes this. The + issue for the tests is #33320. + +commit 4f1ffb5cb1b5163be02aba9bb54a5f872507a17c +Author: Tim Brooks +Date: Fri Aug 31 13:33:14 2018 -0600 + + Mute SmokeTestWatcherWithSecurityIT testsi + + Tests from the SmokeTestWatcherWithSecurityIT suite have been failing + occasionally. This commit mutes all the tests. This is tracked in + +commit 3d82a30fadd035228f29ccc00d6c5bab71e9adf6 +Author: Vladimir Dolzhenko +Date: Fri Aug 31 21:29:06 2018 +0200 + + drop `index.shard.check_on_startup: fix` (#32279) + + drop `index.shard.check_on_startup: fix` + + Relates #31389 + +commit d4f2b5be7de735ff4a5fb360c00ffc1741f3ddc4 +Author: Tal Levy +Date: Fri Aug 31 12:03:49 2018 -0700 + + tracked at + + https://github.com/elastic/elasticsearch/issues/33320 + and + https://github.com/elastic/elasticsearch/issues/30777 + +commit 874ebcb6d48b86dbcf539a5f34b91b4d1b38b6ea +Author: Lisa Cawley +Date: Fri Aug 31 11:56:26 2018 -0700 + + [DOCS] Moves ml folder from x-pack/docs to docs (#33248) + +commit cdeadfc585fbbe9c9c01da590eae89869b823291 +Author: Lisa Cawley +Date: Fri Aug 31 10:50:43 2018 -0700 + + [DOCS] Move rollup APIs to docs (#31450) + +commit 08d0527e25b94eabf402c6f39c587b00ce76a1d1 +Author: Lisa Cawley +Date: Fri Aug 31 10:11:58 2018 -0700 + + [DOCS] Rename X-Pack Commands section (#33005) + +commit 8703d875c0d14c7a723e9a02467bff65714b2f8c +Author: Nhat Nguyen +Date: Fri Aug 31 12:54:12 2018 -0400 + + TEST: Disable soft-deletes in ParentChildTestCase + + Tracked at #33318 + +commit 436d5c4eeec6185e8c745095a68227ea6e46d4b6 +Author: Colin Goodheart-Smithe +Date: Fri Aug 31 17:47:05 2018 +0100 + + Fixes SecurityIntegTestCase so it always adds at least one alias (#33296) + + * Fixes SecurityIntegTestCase so it always adds at least one alias + + `SecurityIntegTestCase.createIndicesWithRandomAliases` could randomly + fail because its not gauranteed that the randomness of which aliases to + add to the `IndicesAliasesRequestBuilder` would always select at least + one alias to add. This change fixes the problem by keeping track of + whether we have added an alias to teh request and forcing the last + alias to be added if no other aliases have been added so far. + + Closes #30098 + Closes #33123e + + * Addresses review comments + +commit 66b164c2a6176b5cbb9e44e650749dbae7bec8d0 +Author: Martijn van Groningen +Date: Fri Aug 31 21:16:06 2018 +0700 + + [CCR] Removed custom follow and unfollow api's reponse classes with AcknowledgedResponse (#33260) + + These response classes did not add any value and in that case just AcknowledgedResponse should be used. + + I also changed the formatting of methods to take one line per parameter in + FollowIndexAction.java and UnfollowIndexAction.java files to make + reviewing diffs in the future easier. + +commit db3d32ce91b02d0de76085a3c47d57e4d21f0e92 +Author: Alpar Torok +Date: Fri Aug 31 16:48:00 2018 +0300 + + Fix pom for build-tools (#33300) + + Looks like `java-gradle-plugin` reconfigures the pom. + Stop using it since we don't publish to Gradle plugin portal. + +commit 3eef74d5d582c19c29d7a297a120004bf4d38930 +Merge: ada3e710f6a a84a20844be +Author: Colin Goodheart-Smithe +Date: Fri Aug 31 14:45:22 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit a84a20844be0bbb39c7e063e33691f8d3e0a8380 +Author: Alpar Torok +Date: Fri Aug 31 16:36:57 2018 +0300 + + Lazy evaluate java9home (#33301) + +commit 0c4b3162be7c2a6bd03a44ab15821ef49a776738 +Author: Andrei Stefan +Date: Fri Aug 31 16:12:01 2018 +0300 + + SQL: test coverage for JdbcResultSet (#32813) + + * Tests for JdbcResultSet + * Added VARCHAR conversion for different types + * Made error messages consistent: they now contain both the type that fails to be converted and the value itself + +commit f6a570880c5c44adea4ad1f35e950c51335d8ac7 +Author: Alpar Torok +Date: Fri Aug 31 16:01:54 2018 +0300 + + Work around to be able to generate eclipse projects (#33295) + + * Work around to be able to generate eclipse projects + + https://github.com/gradle/gradle/issues/6582 + +commit a88f8789a0a9260de0774ee7b76446c2a792294f +Author: Pablo Musa +Date: Fri Aug 31 14:48:55 2018 +0200 + + Highlight that index_phrases only works if no slop is used (#33303) + + Highlight that `index_phrases` only works if no slop is used at query time. + +commit 20e1b5e2fe3e7319e2c908657e484bb49bbf11d7 +Author: Andrei Stefan +Date: Fri Aug 31 13:08:32 2018 +0300 + + Different handling for security specific errors in the CLI. Fix for https://github.com/elastic/elasticsearch/issues/33230 (#33255) + +commit 7345878d33a00ce95eae929f79b2da0b326d182d +Author: David Roberts +Date: Fri Aug 31 08:48:45 2018 +0100 + + [ML] Refactor delimited file structure detection (#33233) + + 1. Use the term "delimited" rather than "separated values" + 2. Use a single factory class with arguments to specify the + delimiter and identification constraints + + This change makes it easier to add support for other + delimiter characters. + +commit 73eb4cbbbe1c38508f6fc303ca300c508952b507 +Author: Costin Leau +Date: Fri Aug 31 10:45:25 2018 +0300 + + SQL: Support multi-index format as table identifier (#33278) + + Extend tableIdentifier to support multi-index format; not just * but + also enumeration and exclusion + + Fix #33162 + +commit ada3e710f6a3ab36f95a5268217be599454d52f2 +Author: Colin Goodheart-Smithe +Date: Fri Aug 31 08:29:44 2018 +0100 + + Renames XPackField.INDEX _LIFCYCLE value to "ilm" (#33270) + + This brings the name in line with everywhere else and means that name + seen on the feature usage and `GET _xpack` APIs will match the plugin + name. + + This change also removes `IndexLifcycle.NAME` since this was only used + to name the scheduler job and that can be done using + `XPackField.INDEX_LIFECYCLE` instead + +commit c6cfa08a615f5b433cf64bef63220f4a22727115 +Author: Armin Braun +Date: Fri Aug 31 08:40:27 2018 +0200 + + MINOR: Remove Dead Code from PathTrie (#33280) + + * The array size checks are redundant since the array sizes + are checked earlier in those methods too + * The removed methods are just not used anywhere + +commit 44ed5f6306038c7f80cbd6ffe403104248145d1e +Author: Alpar Torok +Date: Fri Aug 31 09:31:55 2018 +0300 + + Enable forbiddenapis server java9 (#33245) + +commit 53300670337686f4c89a559213c70c26d90a302f +Merge: 88d8cadf3ff ad4dd086d2c +Author: Nhat Nguyen +Date: Fri Aug 31 00:07:21 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Integrates soft-deletes into Elasticsearch (#33222) + Revert "Integrates soft-deletes into Elasticsearch (#33222)" + Add support for "authorization_realms" (#33262) + +commit 88d8cadf3ffb13026a5c5994a4ec0f981d3f5772 +Author: Nhat Nguyen +Date: Fri Aug 31 00:07:11 2018 -0400 + + Backout IcuTokenizerFactory change + + This change is no longer needed as it's handled in the upstream. + +commit ad4dd086d2cef4df3008fd1a3acffe5735231ce8 +Author: Nhat Nguyen +Date: Thu Aug 30 22:11:23 2018 -0400 + + Integrates soft-deletes into Elasticsearch (#33222) + + This PR integrates Lucene soft-deletes(LUCENE-8200) into Elasticsearch. + Highlight works in this PR include: + + - Replace hard-deletes by soft-deletes in InternalEngine + - Use _recovery_source if _source is disabled or modified (#31106) + - Soft-deletes retention policy based on the global checkpoint (#30335) + - Read operation history from Lucene instead of translog (#30120) + - Use Lucene history in peer-recovery (#30522) + + Relates #30086 + Closes #29530 + + --- + These works have been done by the whole team; however, these individuals + (lexical order) have significant contribution in coding and reviewing: + + Co-authored-by: Adrien Grand + Co-authored-by: Boaz Leskes + Co-authored-by: Jason Tedor + Co-authored-by: Martijn van Groningen + Co-authored-by: Nhat Nguyen + Co-authored-by: Simon Willnauer + +commit 547de71d5991ecee65fcc2ae05cadf95fad717f9 +Author: Nhat Nguyen +Date: Thu Aug 30 23:44:57 2018 -0400 + + Revert "Integrates soft-deletes into Elasticsearch (#33222)" + + Revert to correct co-author tags. + This reverts commit 6dd0aa54f6d0ad44219b3b416438b1be57cb37e5. + +commit 273c82d7c9d70f3378c3e2dafcd2fab851c4308f +Author: Tim Vernum +Date: Fri Aug 31 13:25:27 2018 +1000 + + Add support for "authorization_realms" (#33262) + + Authorization Realms allow an authenticating realm to delegate the task + of constructing a User object (with name, roles, etc) to one or more + other realms. + + E.g. A client could authenticate using PKI, but then delegate to an LDAP + realm. The LDAP realm performs a "lookup" by principal, and then does + regular role-mapping from the discovered user. + + This commit includes: + - authorization_realm support in the pki, ldap, saml & kerberos realms + - docs for authorization_realms + - checks that there are no "authorization chains" + (whereby "realm-a" delegates to "realm-b", but "realm-b" delegates to "realm-c") + + Authorization realms is a platinum feature. + +commit d3f32273ebb0e13f89edd4b39c545d6a0fc79cdd +Merge: 13261996cee 6dd0aa54f6d +Author: Nhat Nguyen +Date: Thu Aug 30 23:09:01 2018 -0400 + + Merge branch 'master' into ccr + +commit 6dd0aa54f6d0ad44219b3b416438b1be57cb37e5 +Author: Nhat Nguyen +Date: Thu Aug 30 22:11:23 2018 -0400 + + Integrates soft-deletes into Elasticsearch (#33222) + + This PR integrates Lucene soft-deletes(LUCENE-8200) into Elasticsearch. + Highlight works in this PR include: + + - Replace hard-deletes by soft-deletes in InternalEngine + - Use _recovery_source if _source is disabled or modified (#31106) + - Soft-deletes retention policy based on the global checkpoint (#30335) + - Read operation history from Lucene instead of translog (#30120) + - Use Lucene history in peer-recovery (#30522) + + Relates #30086 + Closes #29530 + + --- + These works have been done by the whole team; however, these individuals + (lexical order) have significant contribution in coding and reviewing: + + Co-authored-by: Adrien Grand jpountz@gmail.com + Co-authored-by: Boaz Leskes b.leskes@gmail.com + Co-authored-by: Jason Tedor jason@tedor.me + Co-authored-by: Martijn van Groningen martijn.v.groningen@gmail.com + Co-authored-by: Nhat Nguyen nhat.nguyen@elastic.co + Co-authored-by: Simon Willnauer simonw@apache.org + +commit 86feb7713b1903c2f1aca1a1de44c5477bbe40ae +Author: Tal Levy +Date: Thu Aug 30 18:13:50 2018 -0700 + + [MUTE] SmokeTestWatcherWithSecurityIT flaky tests + +commit 39839f97ef8fa4562e2c2e0566867943b0e0de96 +Author: Nhat Nguyen +Date: Thu Aug 30 19:37:20 2018 -0400 + + TEST: Access cluster state directly in assertSeqNos (#33277) + + Some AbstractDisruptionTestCase tests start failing since we enabled + assertSeqNos (in #33130). They fail because the assertSeqNos assertion + queries cluster stats while the cluster is disrupted or not formed yet. + + This commit switches to use the cluster state and shard stats directly + from the test cluster. + + Closes #33251 + +commit 4a958ee2bc2c55754d1a415030e6b7e9d945131c +Author: Tal Levy +Date: Thu Aug 30 15:29:04 2018 -0700 + + fix shrunken index source name metadata usage + +commit 13a0d822d0ef425a9b5f27b10a64e20485017833 +Merge: 246545173a2 bbbb11a698e +Author: Tal Levy +Date: Thu Aug 30 15:04:17 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 8a2d154bad8976af3c1282b7d5dd0b2ceb1c2454 +Author: Lee Hinman +Date: Thu Aug 30 15:05:38 2018 -0600 + + Update serialization versions for custom IndexMetaData backport + +commit 246545173a262fa996a29c9e80c29494339cb1f8 +Author: Gordon Brown +Date: Thu Aug 30 15:51:16 2018 -0600 + + HLRC: Add Put Lifecycle Policy API to HLRC (#33250) + + * Move Lifecycle objects from protcol to core + * HLRC: Add Put Lifecycle API to the HLRC + * Use HLRC Put for ILM integration tests + +commit bbbb11a698ed556de14cc3fcfb365a60493519b9 +Author: Jack Conradson +Date: Thu Aug 30 14:33:32 2018 -0700 + + Painless: Fix Bindings Bug (#33274) + + When the change was made to the format for in the whitelist for bindings, parameters from + both the constructor and the method were combined into a single list instead of separate + lists. The check for method parameters was being executed from the start of the combined + list rather than the correct position. The tests for bindings used a constructor and a method + that only used the int types so this was not caught. The test has been changed to also use + a double type and this issue is fixed. + +commit 83c3d7a6cfbb469888bdf1429489a7dc07db6020 +Author: Costin Leau +Date: Fri Aug 31 00:13:03 2018 +0300 + + SQL: prevent duplicate generation for repeated aggs (#33252) + + Prevent generation of duplicate aggs caused by repetitive functions, + leading to invalid query. + + Fix #30287 + +commit 001b78f704d25d01d4c28a4a756c536321c8dc8a +Author: Igor Motov +Date: Fri Aug 31 01:27:00 2018 +0530 + + Replace IndexMetaData.Custom with Map-based custom metadata (#32749) + + This PR removes the deprecated `Custom` class in `IndexMetaData`, in favor + of a `Map` that is used to store custom index + metadata. As part of this, there is now no way to set this metadata in a + template or create index request (since it's only set by plugins, or dedicated + REST endpoints). + + The `Map` is intended to be a namespaced `Map` (`DiffableStringMap` implements `Map`, so the signature + is more like `Map>`). This is so we can do things + like: + + ``` java + Map ccrMeta = indexMetaData.getCustom("ccr"); + ``` + + And then have complete control over the metadata. This also means any + plugin/feature that uses this has to manage its own BWC, as the map is just + serialized as a map. It also means that if metadata is put in the map that isn't + used (for instance, if a plugin were removed), it causes no failures the way + an unregistered `Setting` would. + + The reason I use a custom `DiffableStringMap` here rather than a plain + `Map` is so the map can be diffed with previous cluster state + updates for serialization. + + Supersedes #32683 + +commit af2eaf2a6c93aa80671b9e89d583509f75644dcd +Author: Simon Willnauer +Date: Thu Aug 30 21:08:35 2018 +0200 + + Remove usage of `index.shrink.source.*` in 7.x (#33271) + + We cut over to `index.resize.source.*` but still have these constants + being public in `IndexMetaData`. Those Settings and constants are not needed + in 7.x while we still need to keep the keys known to private settings since + they might be part of the index settings of old indices. We can remove that + in 8.0. Yet, we should remove the settings to make sure they are not used again. + +commit 4257d05869683e8984db854ae2fb33b06598aa4f +Author: Jake Landis +Date: Thu Aug 30 13:46:52 2018 -0500 + + Revert "HLRC: Add ILM Status to HLRC" + + This reverts commit 5a705e96645b285477af438a149f96ead3fa8a60. + + (accidental push to wrong branch) + +commit 5a705e96645b285477af438a149f96ead3fa8a60 +Author: Jake Landis +Date: Thu Aug 30 13:38:04 2018 -0500 + + HLRC: Add ILM Status to HLRC + + Adds support for the Index Lifecycle Management Status to the Java + High-Level REST Client. + + Relates to #33100 + +commit 07faa0b06a7d8ed8cd84ef234692c4413994e088 +Author: Nhat Nguyen +Date: Thu Aug 30 14:26:32 2018 -0400 + + TEST: Mute testMonitorClusterHealth + + Tracked at #32299 + +commit d0630093cdeada66904e648824a42f5e4950b895 +Author: Jim Ferenczi +Date: Thu Aug 30 18:07:58 2018 +0200 + + Fix serialization of empty field capabilities response (#33263) + + Fix serialization of empty field capabilities response + + When no response are required (no indices match the requested patterns) the + empty response throws an NPE in the transport serialization (writeTo). + +commit 1404dd2a42c88fe72159587429f59f96251e6d41 +Author: Jim Ferenczi +Date: Thu Aug 30 15:15:50 2018 +0200 + + Fix nested _source retrieval with includes/excludes (#33180) + + If an exclude or an include clause removes an entry to a nested field in the original source at query time, + the creation of nested hits fails with an NPE. This change fixes this exception and replaces the nested document + source with an empty map. + + Closes #33163 + Closes #33170 + +commit 13261996cee0225f7ac1191317e29b2c3d155841 +Author: Nhat Nguyen +Date: Thu Aug 30 07:55:13 2018 -0400 + + Add NoOps to Lucene for failed delete ops (#33217) + + Today we add a NoOp to Lucene and translog if we fail to process an + indexing operation. However, we are only adding NoOps to translog for + delete operations. In order to have a complete history in Lucene, we + should add NoOps of failed delete operations to both Lucene and translog. + + Relates #29530 + +commit 557eabf7b5e8461f9566224dee6b23c6f8da266b +Author: Ioannis Kakavas +Date: Thu Aug 30 13:59:19 2018 +0300 + + [DOCS] TLS file resources are reloadable (#33258) + + Make clearer that file resources that are used as key trust material + are polled and will be reloaded upon modification. + +commit b6f762d131455fa16a7c5699372eabc42a1e2b61 +Author: Alexander Reelsen +Date: Thu Aug 30 10:53:01 2018 +0200 + + Watcher: Ensure TriggerEngine start replaces existing watches (#33157) + + This commit ensures that when `TriggerService.start()` is called, + we ensure in the trigger engine implementations that current watches are + removed instead of adding to the existing ones in + `TickerScheduleTriggerEngine.start()` + + Two additional minor fixes, where the result remains the same but less code gets executed. + + 1. If the node is not a data node, we forgot to set the status to + STARTING when watcher is being started. This should not be a big issue, + because a non-data node does not spent a lot of time loading as there + are no watches which need loading. + 2. If a new cluster state came in during a reload, we had two checks in + place to abort loading the current one. The first one before we load all + the watches of the local node and the second before watcher is starting + with those new watches. Turned out that the first check was not + returning, which meant we always tried to load all the watches, and then + would fail on the second check. This has been fixed here. + +commit 5cf6e0d4bc9bde825c67207c75d7a9c0b6368e68 +Author: Alpar Torok +Date: Thu Aug 30 11:41:39 2018 +0300 + + Ignore module-info in jar hell checks (#33011) + + * Ignore module-info in JarHell checks + * Add unit test + * integration test to test that jarhell is ran with precommit + +commit 51cbc611353b50126c003c88a27303f0b9ddd80f +Author: David Turner +Date: Thu Aug 30 09:38:23 2018 +0100 + + Fix docs build after #33241 + + Recently-merged PR #33241 broke the docs build, and this fixes it. + +commit c31c51dc801f48de2c48d4fcb3e3f63216158cc6 +Author: Albert Zaharovits +Date: Thu Aug 30 10:32:08 2018 +0300 + + [DOC] Repository GCS ADC not supported (#33238) + + Make it clear that automatic default credentials (ADC) + is not supported for the repository-gcs plugin. + "Service Account" method is the only alternative + to authn requests to Google Cloud Storage. + +commit 9c541b8f729038fa063f0a41bbc67f64da8352ea +Author: Alpar Torok +Date: Thu Aug 30 10:20:38 2018 +0300 + + Upgrade to latest Gradle 4.10 (#32801) + + Upgrade to Gradle 4.10 + +commit f097446066fa6ee9e09e7e86086194c9f6bdb0a3 +Author: Alpar Torok +Date: Thu Aug 30 10:17:42 2018 +0300 + + Fix/30904 cluster formation part2 (#32877) + + Gradle integration for the Cluster formation plugin with ref counting + +commit 214652d4af8188d4ba872626eeea3bcdff7096f0 +Author: Ioannis Kakavas +Date: Thu Aug 30 09:13:28 2018 +0300 + + [TESTS] Pin MockWebServer to TLS1.2 (#33127) + + Ensure that the SSLConfigurationReloaderTests can run with JDK 11 + by pinning the Server TLS version to TLS1.2. This can be revisited + while tackling the effort to full support TLSv1.3 in + https://github.com/elastic/elasticsearch/issues/32276 + + Resolves #32124 + +commit 47859e56ac31f2c453a9eed6f5217171e0bda7f5 +Author: David Turner +Date: Thu Aug 30 06:43:04 2018 +0100 + + Move file-based discovery to core (#33241) + + Today we support a static list of seed hosts in core Elasticsearch, and allow a + dynamic list of seed hosts to be provided via a file using the `discovery-file` + plugin. In fact the ability to provide a dynamic list of seed hosts is + increasingly useful, so this change moves this functionality to core + Elasticsearch to avoid the need for a plugin. + + Furthermore, in order to start up nodes in integration tests we currently + assign a known port to each node before startup, which unfortunately sometimes + fails if another process grabs the selected port in the meantime. By moving the + `discovery-file` functionality into the core product we can use it to avoid + this race. + + This change also moves the expected path to the file from + `$ES_PATH_CONF/discovery-file/unicast_hosts.txt` to + `$ES_PATH_CONF/unicast_hosts.txt`. An example of this file is not included in + distributions. + + For BWC purposes the plugin still exists, but does nothing more than create the + example file in the old location, and issue a warning when it is used. We also + continue to support the old location for the file, but warn about its + deprecation. + + Relates #29244 + Closes #33030 + +commit f0635870832dbe1d06ef346f700cdda0feb4f045 +Author: Jay Modi +Date: Wed Aug 29 21:44:33 2018 -0600 + + HLRC: add client side RefreshPolicy (#33209) + + With the switch to client side request and response objects, we need a + client side version of RefreshPolicy. This change adds a client side + version of RefreshPolicy along with a method to add it to the + parameters of a request. The existing method to add + WriteRequest.RefreshPolicy to the parameters of a request is now + deprecated. + +commit 6fd971040e6bacde60ca3b617951f4e316c32ed9 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Thu Aug 30 12:08:29 2018 +1000 + + [Kerberos] Add unsupported languages for tests (#33253) + + Ran for all locales in system to find locales which caused + problems in tests due to incorrect generalized time handling + in simple kdc ldap server. + + Closes#33228 + +commit cc4d7059bfda42042e4e320c7c2bb228c0fe4691 +Author: Armin Braun +Date: Thu Aug 30 03:46:39 2018 +0200 + + Ingest: Add conditional per processor (#32398) + + * Ingest: Add conditional per processor + * closes #21248 + +commit d93b2a2e9a68ae4defd5949fcc5eb945a26395bf +Author: Zachary Tong +Date: Wed Aug 29 17:10:00 2018 -0400 + + [Rollup] Only allow aggregating on multiples of configured interval (#32052) + + We need to limit the search request aggregations to whole multiples + of the configured interval for both histogram and date_histogram. + Otherwise, agg buckets won't overlap with the rolled up buckets + and the results will be incorrect. + + For histogram, the validation is very simple: request must be >= the config, + and modulo evenly. + + Dates are more tricky. + - If both request and config are fixed dates, we can convert to millis + and treat them just like the histo + - If both are calendar, we make sure the request is >= the config with + a static lookup map that ranks the calendar values relatively. All + calendar units are "singles", so they are evenly divisible already + - We disallow any other combination (one fixed, one calendar, etc) + +commit 13880bd8c1ecac163a80e46a8fea943e6be8516c +Author: Alexander Reelsen +Date: Wed Aug 29 22:42:08 2018 +0200 + + Watcher: Reload properly on remote shard change (#33167) + + When a node dies that carries a watcher shard or a shard is relocated to + another node, then watcher needs not only trigger a reload on the node + where the shard relocation happened, but also on other nodes where + copies of this shard, as different watches may need to be loaded. + + This commit takes the change of remote nodes into account by not only + storing the local shard allocation ids in the WatcherLifeCycleService, + but storing a list of ShardRoutings based on the local active shards. + + This also fixes some tests, which had a wrong assumption. Using + `TestShardRouting.newShardRouting` in our tests for cluster state + creation led to the issue of always creating new allocation ids which + implicitely lead to a reload. + +commit 0f22dbb1cce49e7a1dbca76e4f684140c57fdb09 +Author: Jason Tedor +Date: Wed Aug 29 15:56:13 2018 -0400 + + Apply settings filter to get cluster settings API (#33247) + + Some settings have filters applied to them and we use this in logs and + the get nodes info API. For consistency, we should apply this in the get + cluster settings API too. + +commit 92bd7242a36be6511f463722bedbe7a85f176f2e +Author: Matt Weber +Date: Wed Aug 29 12:19:58 2018 -0700 + + Fix classpath security checks for external tests. (#33066) + + This commit checks that when we manually add a class to + the codebase map, that it does in-fact not exist on the classpath + in a jar. This will only be true if we are using the test framework + externally such as when a user develops a plugin. + +commit ba8d4eba8b227a153550ce5a2c30f83a1f5ca1f8 +Author: Gordon Brown +Date: Wed Aug 29 13:14:46 2018 -0600 + + HLRC: Add Lifecycle Policy delete to the HLRC (#33142) + + Adds support for Lifecycle Policy deletion to the Java High-Level Rest Client. + + Relates to #33100 + +commit cfc003d4855601b1f1bf2be522f1c6087feeb229 +Author: Hendrik Muhs +Date: Wed Aug 29 20:28:21 2018 +0200 + + [Rollup] Re-factor Rollup Indexer into a generic indexer for re-usability (#32743) + + This extracts a super class out of the rollup indexer called the AsyncTwoPhaseIterator. + The implementor of it can define the query, transformation of the response, + indexing and the object to persist the position/state of the indexer. + + The stats object used by the indexer to record progress is also now abstract, allowing + the implementation provide custom stats beyond what the indexer provides. It also + allows the implementation to decide how the stats are presented (leaves toXContent() + up to the implementation). + + This should allow new projects to reuse the search-then-index persistent task that Rollup + uses, but without the restrictions/baggage of how Rollup has to work internally to + satisfy time-based rollups. + +commit 5632e31c747bb171c3a7b205b5f95ed6b14e4a63 +Merge: 41c7fc8d375 b52818ec6f0 +Author: Nhat Nguyen +Date: Wed Aug 29 12:30:24 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Painless: Add Bindings (#33042) + Update version after client credentials backport + Fix forbidden apis on FIPS (#33202) + Remote 6.x transport BWC Layer for `_shrink` (#33236) + Test fix - Graph HLRC tests needed another field adding to randomisation exception list + HLRC: Add ML Get Records API (#33085) + [ML] Fix character set finder bug with unencodable charsets (#33234) + TESTS: Fix overly long lines (#33240) + Test fix - Graph HLRC test was missing field name to be excluded from randomisation logic + Remove unsupported group_shard_failures parameter (#33208) + Update BucketUtils#suggestShardSideQueueSize signature (#33210) + Parse PEM Key files leniantly (#33173) + INGEST: Add Pipeline Processor (#32473) + Core: Add java time xcontent serializers (#33120) + Consider multi release jars when running third party audit (#33206) + Update MSI documentation (#31950) + HLRC: create base timed request class (#33216) + [DOCS] Fixes command page titles + HLRC: Move ML protocol classes into client ml package (#33203) + Scroll queries asking for rescore are considered invalid (#32918) + Painless: Fix Semicolon Regression (#33212) + ingest: minor - update test to include dissect (#33211) + Switch remaining LLREST usage to new style Requests (#33171) + HLREST: add reindex API (#32679) + +commit b52818ec6f0dbc39090554d51317bff31ea3dec1 +Author: Jack Conradson +Date: Wed Aug 29 09:24:56 2018 -0700 + + Painless: Add Bindings (#33042) + + Add bindings that allow some specialized methods to store permanent state between script executions. + +commit 6daf8115d6b0b7a4b6e11230fe25dcce11bffa4a +Author: jaymode +Date: Wed Aug 29 09:02:32 2018 -0600 + + Update version after client credentials backport + + This commit changes the serialization version from V_7_0_0_alpha1 to + V_6_5_0 for the create token request and response with a client + credentials grant type. The client credentials work has now been + backported to 6.x. + + Relates #33106 + +commit 3828ec60f5c358cb9f968488ccdbbde697d66514 +Author: Alpar Torok +Date: Wed Aug 29 17:43:40 2018 +0300 + + Fix forbidden apis on FIPS (#33202) + + - third party audit detects jar hell with JDK so we disable it + - jdk non portable in forbiddenapis detects classes being used from the + JDK ( for fips ) that are not portable, this is intended so we don't + scan for it on fips. + - different exclusion rules for third party audit on fips + + Closes #33179 + +commit 6a0d4b4a773d2c3d463b9807f400f911932fe51e +Author: Simon Willnauer +Date: Wed Aug 29 16:43:13 2018 +0200 + + Remote 6.x transport BWC Layer for `_shrink` (#33236) + + The shrink action was renamed to `_resize` with the addition + or split. This bwc layer is unnecessary on 7.x since 6.latest will + always use the resize action. + +commit 454ce99b01ceb7a47e2652e783a9c833c8dbec30 +Merge: 9d157963d45 e95c2afe3c2 +Author: Gordon Brown +Date: Wed Aug 29 08:28:23 2018 -0600 + + Merge branch 'master' into index-lifecycle + +commit e95c2afe3c20da62a71c6deefbf9ecbac5f9be6a +Author: markharwood +Date: Wed Aug 29 15:19:26 2018 +0100 + + Test fix - Graph HLRC tests needed another field adding to randomisation exception list + + Related to #33231 + +commit a5b34c75b08f4cb22ed13e52d250da9418c560df +Author: Dimitris Athanasiou +Date: Wed Aug 29 15:03:58 2018 +0100 + + HLRC: Add ML Get Records API (#33085) + + Relates #29827 + +commit 22415fa2de1d7d07cea7dd5e7263eb1ed4270503 +Author: David Roberts +Date: Wed Aug 29 14:56:02 2018 +0100 + + [ML] Fix character set finder bug with unencodable charsets (#33234) + + Some character sets cannot be encoded and this was tripping + up the binary data check in the ML log structure character + set finder. + + The fix is to assume that if ICU4J identifies that some bytes + correspond to a character set that cannot be encoded and those + bytes contain zeroes then the data is binary rather than text. + + Fixes #33227 + +commit dd1956cf192c958fad250f43633dea5c30493822 +Author: Armin Braun +Date: Wed Aug 29 15:49:35 2018 +0200 + + TESTS: Fix overly long lines (#33240) + +commit 63b2db1d84dd0b31433842600ff278951a0a3681 +Author: markharwood +Date: Wed Aug 29 14:13:30 2018 +0100 + + Test fix - Graph HLRC test was missing field name to be excluded from randomisation logic + + Closes #33231 + +commit 49109187e2adb514d90b017e3ace2753f93be3db +Author: Luca Cavanna +Date: Wed Aug 29 14:05:41 2018 +0200 + + Remove unsupported group_shard_failures parameter (#33208) + + We have had support for the `group_shard_failures` parameter in our code for a while, since we introduced failures grouping. When we introduced validation of parameters at REST, we seem to have forgotten to expose such parameter. Given that the parameter is effectively not supported for many months now, that no user has complained about that and that grouping is the expected behaviour, this commit removes support for the parameter. + +commit 034fdbca28c015840b14be8c744dd895090f1870 +Author: Luca Cavanna +Date: Wed Aug 29 13:51:54 2018 +0200 + + Update BucketUtils#suggestShardSideQueueSize signature (#33210) + + `BucketUtils#suggestShardSideQueueSize` used to calculate the shard_size based on the number of shards. It returns now a different value only based on whether we are querying a single shard or multiple shards. This commit replaces the numberOfShards argument with a boolean that tells whether we are querying a single shard or not. + +commit 8c57d4af6a824ddf9ec8e5329a8c33bac27a8ed3 +Author: Ioannis Kakavas +Date: Wed Aug 29 12:35:31 2018 +0300 + + Parse PEM Key files leniantly (#33173) + + Allow for extra non-whitespace before the Header of PEM encoded + key files. + + Resolves #33168 + +commit f690b492e7855609d59f836b8d5aef317b131ff7 +Author: Armin Braun +Date: Wed Aug 29 11:03:10 2018 +0200 + + INGEST: Add Pipeline Processor (#32473) + + * INGEST: Add Pipeline Processor + + * Adds Processor capable of invoking other pipelines + * Closes #31842 + +commit 48b388ce82d0fb43709b7809ce3166b17b1f16d0 +Author: Alexander Reelsen +Date: Wed Aug 29 10:00:16 2018 +0200 + + Core: Add java time xcontent serializers (#33120) + + This ensures that the java time class exposed by painless have proper + serialization/string representations. + + Closes #31853 + +commit f29f0af7bca1011788718f7f9e8e88136cc6dfa5 +Author: Alpar Torok +Date: Wed Aug 29 09:53:04 2018 +0300 + + Consider multi release jars when running third party audit (#33206) + + Exclude classes meant for newer versions than what we are auditing against, those classes won't be found. There's no reason to exclude JDK classes from newer versions, with this PR, we will not extract them in the first place. + +commit 2dc4a5bb56070db97446ef324e30b7aef1671471 +Author: Stuart Cam +Date: Wed Aug 29 14:34:54 2018 +1000 + + Update MSI documentation (#31950) + +commit 41c7fc8d375f89045f6a9c16bb8af6a2ce4af3e2 +Author: Martijn van Groningen +Date: Wed Aug 29 10:54:58 2018 +0700 + + [CCR] Introduce leader index name & last fetch time stats to stats api response (#33155) + +commit 1be3dd5504a362cef24a13dafb86d4d7db15fa26 +Author: Michael Basnight +Date: Tue Aug 28 17:23:41 2018 -0500 + + HLRC: create base timed request class (#33216) + + There are many requests that allow the user to set a few timeouts + on. This class will allow requests impl'd in HLRC to extend from, and + allow users to set those values without significant work to add them to + every request. + +commit 1cbde721dc1caf67f4903624549572e135cb365f +Author: lcawl +Date: Tue Aug 28 14:34:39 2018 -0700 + + [DOCS] Fixes command page titles + +commit 5697d93cbf078dca7a112aa02e222504a492a861 +Author: Dimitris Athanasiou +Date: Tue Aug 28 20:53:31 2018 +0100 + + HLRC: Move ML protocol classes into client ml package (#33203) + + * HLRC: Move ML protocol classes into client ml package + + * Do not use log4j deprecation handler + + * JavaDoc should refer to correct Job path + +commit 84b61d073895e45fc2717812798d7f100f48402d +Author: Mark Tozzi +Date: Tue Aug 28 15:48:23 2018 -0400 + + Scroll queries asking for rescore are considered invalid (#32918) + + This PR changes our behavior from silently ignoring rescore in a scroll query to instead report to the user that such a query is invalid. + + Closes #31775 + +commit a381749aac3857e4c186f709ec35adef897a4a6f +Author: Jack Conradson +Date: Tue Aug 28 12:40:41 2018 -0700 + + Painless: Fix Semicolon Regression (#33212) + + Trailers (statements following something like an if statement) that don't use brackets currently require a semicolon even if they're the last statement. This is a regression caused by (#29566) and noted by (#33193). This change fixes the regression and adds a test for the broken case. + +commit e9b0807c674b6494a82e3fbdca826d9818985bbd +Author: Jake Landis +Date: Tue Aug 28 11:55:04 2018 -0700 + + ingest: minor - update test to include dissect (#33211) + + This change also includes placing the bytes processor in the correct + order (helps to avoid merge conflict when back patching processors) + +commit 6c8f5688083ba6198c1084eeb2ef75f3a24d1f7d +Author: Nik Everett +Date: Tue Aug 28 14:20:14 2018 -0400 + + Switch remaining LLREST usage to new style Requests (#33171) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. In a + long series of PRs I've changed all of the old style requests that I + could find with `grep`. In this PR I change all requests that I could + find by *removing* the deprecated methods. Since this is a non-trivial + change I do not include actually removing the deprecated requests. I'll + do that in a follow up. But this should be the last set of usage + removals before the actual deprecated method removal. Yay! + +commit c42dc7789632fffd5570a85796eb200535e31746 +Merge: e39689a1983 353112a033f +Author: Nhat Nguyen +Date: Tue Aug 28 13:56:38 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [Rollup] Better error message when trying to set non-rollup index (#32965) + HLRC: Use Optional in validation logic (#33104) + Remove unused User class from protocol (#33137) + ingest: Introduce the dissect processor (#32884) + [Docs] Add link to es-kotlin-wrapper-client (#32618) + [Docs] Remove repeating words (#33087) + Minor spelling and grammar fix (#32931) + Remove support for deprecated params._agg/_aggs for scripted metric aggregations (#32979) + Watcher: Simplify finding next date in cron schedule (#33015) + Run Third party audit with forbidden APIs CLI (part3/3) (#33052) + Fix plugin build test on Windows (#33078) + HLRC+MINOR: Remove Unused Private Method (#33165) + Remove old unused test script files (#32970) + Build analysis-icu client JAR (#33184) + Ensure to generate identical NoOp for the same failure (#33141) + ShardSearchFailure#readFrom to set index and shardId (#33161) + +commit 7f5e29ddb21ee5935852633f0dff483c8fbd34b2 +Author: Sohaib Iftikhar +Date: Tue Aug 28 19:02:23 2018 +0200 + + HLREST: add reindex API (#32679) + + Adds the reindex API to the high level REST client. + +commit e39689a1983429596072ac37b3a4de43bc6ca3d1 +Author: Nhat Nguyen +Date: Tue Aug 28 12:32:09 2018 -0400 + + Send only ops after checkpoint in file-based recovery with soft-deletes (#33190) + + Today a file-based recovery will replay all existing translog operations + from the primary on a replica so that that replica can have a full + history in translog as the primary. However, with soft-deletes enabled, + we should not do it because: + + 1. All operations before the local checkpoint of the safe commit exist in + the commit already. + + 2. The number of operations before the local checkpoint may be considerable + and requires a significant amount of time to replay on a replica. + + Relates #30522 + Relates #29530 + +commit 353112a033fce837db96c19a46ea706d8a33f2fe +Author: Zachary Tong +Date: Tue Aug 28 11:50:35 2018 -0400 + + [Rollup] Better error message when trying to set non-rollup index (#32965) + + We don't allow the user to configure a rollup index against an + existing index, but the exceptions that we return are not clear about + that. They indicate issues with metadata, instead of stating + the real reason (not allowed to use a non-rollup index to store + rollup data). + + This makes the exception better, and adds a bit more testing + +commit 5f0f990afd670ac20f8e8810705d23fc2f4aff74 +Author: Michael Basnight +Date: Tue Aug 28 10:33:18 2018 -0500 + + HLRC: Use Optional in validation logic (#33104) + + The Validatable class comes from an old class in server code, that + assumed null was returned in the event of validation having no + errors. This commit changes that to use Optional, which is cleaner than + passing around null objects. + +commit 1e11b05b58e09f47dfbd155bdfca74399c054ff0 +Author: Jay Modi +Date: Tue Aug 28 08:55:29 2018 -0600 + + Remove unused User class from protocol (#33137) + + This commit removes the unused User class from the protocol project. + This class was originally moved into protocol in preparation for moving + more request and response classes, but given the change in direction + for the HLRC this is no longer needed. Additionally, this change also + changes the package name for the User object in x-pack/plugin/core to + its original name. + +commit 9d157963d45c5bcb748de7a2e061a48a00e9c7f4 +Author: Colin Goodheart-Smithe +Date: Tue Aug 28 15:47:47 2018 +0100 + + change setup policy note + +commit dbf6ba7e21a0d21f5b8b3e33f1c6ce269c1e4489 +Author: Colin Goodheart-Smithe +Date: Tue Aug 28 15:47:01 2018 +0100 + + Adds docs for how to set up a policy (#32987) + + * Adds docs for how to set up a policy + + * Fix failing docs test + +commit e2b931e80b8b4134f94e6eef8bfb9cfe31a571a5 +Author: Nhat Nguyen +Date: Tue Aug 28 10:44:15 2018 -0400 + + Use Lucene history in primary-replica resync (#33178) + + This commit makes primary-replica resyncer use Lucene as the source of + history operation instead of translog if soft-deletes is enabled. With + this change, we no longer expose translog snapshot directly in IndexShard. + + Relates #29530 + +commit 79b507dbf58051aad2e4ffea00d443e25595088d +Author: Jake Landis +Date: Tue Aug 28 07:11:20 2018 -0700 + + ingest: Introduce the dissect processor (#32884) + + * ingest: Introduce the dissect processor + + The ingest node dissect processor is an alternative to Grok + to split a string based on a pattern. Dissect differs from + Grok such that regular expressions are not used to split the + string. + + Dissect can be used to parse a source text field with a + simpler pattern, and is often faster the Grok for basic string + parsing. This processor uses the dissect library which + does most of the work. + +commit 5954354e6200f8fa1d3304e5bff304adf2b17aa8 +Author: Jason Tedor +Date: Tue Aug 28 08:53:45 2018 -0400 + + Fix ShardFollowNodeTask.Status equals and hash code (#33189) + + These were broken when fetch exceptions were introduced to the status + object but equals and hash code were not updated then. This commit + addresses that. + +commit d8a1b7cb177aa8b8732e5ef072cb139695dc6d1c +Author: Nhat Nguyen +Date: Tue Aug 28 08:48:42 2018 -0400 + + Make soft-deletes settings final (#33172) + + For now, we do not support changing the soft-deletes setting even with + closed indices. Therefore we should make it a final setting. + + Relates #29530 + +commit d7e4a493554037559bbee48a71a97c14ec793dda +Author: Jilles van Gurp +Date: Tue Aug 28 14:25:55 2018 +0200 + + [Docs] Add link to es-kotlin-wrapper-client (#32618) + + ES Kotlin Wrapper client is a library that wraps the official Highlevel Elasticsearch HTTP client for Java. + +commit b7c0d2830ab4f5e49de1b3803bbe5bfb09594f6a +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Tue Aug 28 13:16:43 2018 +0200 + + [Docs] Remove repeating words (#33087) + +commit 525cda03314b4b37f805b7b8522bb72b7fded13d +Author: Dominik +Date: Tue Aug 28 20:48:47 2018 +1000 + + Minor spelling and grammar fix (#32931) + +commit cd91992c89f98ed89d6d5a8d8605f697b3d4014c +Author: Jason Tedor +Date: Tue Aug 28 06:06:22 2018 -0400 + + Only fetch mapping updates when necessary (#33182) + + Today we fetch the mapping from the leader and apply it as a mapping + update whenever the index metadata version on the leader changes. Yet, + the index metadata can change for many reasons other than a mapping + update (e.g., settings updates, adding an alias, or a replica being + promoted to a primary among many other reasons). This commit builds on + the addition of a mapping version to the index metadata to only fetch + mapping updates when the mapping version increases. This reduces the + number of these fetches and application of mappings on the follower to + the bare minimum. + +commit 9d92a87ae6579c00cbf1bf958549599a2a3cc454 +Author: Jonathan Little +Date: Tue Aug 28 01:27:43 2018 -0700 + + Remove support for deprecated params._agg/_aggs for scripted metric aggregations (#32979) + +commit 19ef41ee82a3e801e6380eaf35f79b004321f5eb +Author: Alexander Reelsen +Date: Tue Aug 28 09:06:43 2018 +0200 + + Watcher: Simplify finding next date in cron schedule (#33015) + + The code introduced in 3fa36807f87ad90af593e86f9ed843ced3260973 to fix + an issue with crons always returning -1 was not very readable. This + implementation uses streams to improve readability. + +commit 2cc611604f16cbedf94fb218da9fe29da7727868 +Author: Alpar Torok +Date: Tue Aug 28 10:03:30 2018 +0300 + + Run Third party audit with forbidden APIs CLI (part3/3) (#33052) + + The new implementation is functional equivalent with the old, ant based one. + It parses task standard error to get the missing classes and violations in the same way. + I considered re-using ForbiddenApisCliTask but Gradle makes it hard to build inheritance with tasks that have task actions , since the order of the task actions can't be controlled. + This inheritance isn't dully desired either as the third party audit task is much more opinionated and we don't want to expose some of the configuration. + We could probably extract a common base class without any task actions, but probably more trouble than it's worth. + + Closes #31715 + +commit 71d5c66fd384aaebfb83e0b24920df984eb9f09b +Author: Alpar Torok +Date: Tue Aug 28 10:00:11 2018 +0300 + + Fix plugin build test on Windows (#33078) + + Fix plugin build test on Windows + +commit daee8bd13375a79994d48350535cc90dff7f23b1 +Author: Armin Braun +Date: Tue Aug 28 07:44:01 2018 +0200 + + HLRC+MINOR: Remove Unused Private Method (#33165) + + * This one seems to be unused since 92eb32477 + +commit 29ba143e2cac93e18a576e9501898303f256b910 +Author: Jonathan Little +Date: Mon Aug 27 16:29:50 2018 -0700 + + Remove old unused test script files (#32970) + +commit 5b11df9c351dc7b0fc2124c9f7fced8d8d52dfca +Author: Nhat Nguyen +Date: Mon Aug 27 18:33:57 2018 -0400 + + DOCS: Make ellipsis optional in /cat/thread_pool (#33186) + + The fix proposed in #31442 fails with the oss distro because the added + 3dots does not match anything with the default oss while a 3dots + expression requires matching at least one thread pool. + + This change makes an ellipsis optional so the thread_pool list can match + both the oss distro (without ccr) and default distro (with ccr). + + Relates #31442 + +commit 50368656ee2cd7cacb7c029ce46743c523b09585 +Merge: cfe0acc83c9 5937e499e1b +Author: Gordon Brown +Date: Mon Aug 27 15:35:19 2018 -0600 + + Merge branch 'master' into index-lifecycle + +commit cfe0acc83c94a0068666a85b532c56aef2e96835 +Author: Tal Levy +Date: Mon Aug 27 14:25:27 2018 -0700 + + separate out IndexLifecycleService cluster-state change concerns (#33033) + + Changes to the IndexLifecycleService were necessary since relying on + ClusterChangedEvents for a full picture of the cluster state's settings was + a mistake. It is not necessary that these events hold all settings, especially ones + that are set at node start-up. + + Changes to main include: + + - move poll interval updates to a SettingsUpdateConsumer + - move scheduler start/stop to a localMasterNodeListener + - keep triggerPolicies in clusterChanged + + Changes to tests include: + + - removal of some low-level state transition checks in the Service that no longer make sense + since the changes are unconditionally specified in the appropriate listeners + - add integration tests for poll-interval updates + - add integration test assertions for verifying scheduler is started up correctly + +commit 5937e499e1b7f2dd15a1ce7bc21f4ffa7d9f2cbc +Author: Jason Tedor +Date: Mon Aug 27 16:41:32 2018 -0400 + + Build analysis-icu client JAR (#33184) + + This plugin needs to be able to be installed client side because it + contains doc values formats that can be returned to the transport + client. To keep this simple for developers, we publish the client JAR to + Maven so that they can depend on the plugin in their POM and install the + plugin there. + +commit 014b3236dcb6f2942945c360b343dccac605b466 +Author: Nhat Nguyen +Date: Mon Aug 27 15:59:42 2018 -0400 + + Ensure to generate identical NoOp for the same failure (#33141) + + We generate slightly different NoOps in InternalEngine and + TransportShardBulkAction for the same failure. + + 1. InternalEngine uses Exception#getFailure to generate a message + without the class name: newOp [NoOp{seqNo=1, primaryTerm=1, + reason='Contexts are mandatory in context enabled completion field + [suggest_context]'}]. + + 2. TransportShardBulkAction uses Exception#toString to generate a + message with the class name: NoOp{seqNo=1, primaryTerm=1, + reason='java.lang.IllegalArgumentException: Contexts are mandatory in + context enabled completion field [suggest_context]'}. + + If a write operation fails while a replica is recovering, that replica + will possibly receive two different NoOps: one from recovery and one + from replication. These two different NoOps will trip + TranslogWriter#assertNoSeqNumberConflict assertion. + + This commit ensures that we generate the same Noop for the same failure. + + Closes #32986 + +commit ed0571e16caac689a77362bfe4c45c79dcba4d82 +Author: Luca Cavanna +Date: Mon Aug 27 20:31:27 2018 +0200 + + ShardSearchFailure#readFrom to set index and shardId (#33161) + + As part of recent changes made to `ShardOperationFailedException` we introduced `index` and `shardId` members to the base class, but the subclasses are entirely responsible for the serialization of such fields. In the case of `ShardSearchFailure`, we have an additional `SearchShardTarget` instance member which also holds the index and the shardId, hence they get serialized as part of `SearchShardTarget` itself. When de-serializing a `ShardSearchFailure` though, we need to remember to also set the parent class `index` and `shardId` fields otherwise they get lost + + Relates to #32640 + +commit 0e5d42ca38a174153d64cd8fea91afe5e8ce7e4e +Merge: 47e9e72df23 318df2a107f +Author: Jason Tedor +Date: Mon Aug 27 13:39:38 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Adjust BWC version on mapping version + Token API supports the client_credentials grant (#33106) + Build: forked compiler max memory matches jvmArgs (#33138) + Introduce mapping version to index metadata (#33147) + SQL: Enable aggregations to create a separate bucket for missing values (#32832) + Fix grammar in contributing docs + SECURITY: Fix Compile Error in ReservedRealmTests (#33166) + APM server monitoring (#32515) + Support only string `format` in date, root object & date range (#28117) + [Rollup] Move toBuilders() methods out of rollup config objects (#32585) + Fix forbiddenapis on java 11 (#33116) + Apply publishing to genreate pom (#33094) + Have circuit breaker succeed on unknown mem usage + Do not lose default mapper on metadata updates (#33153) + Fix a mappings update test (#33146) + Reload Secure Settings REST specs & docs (#32990) + Refactor CachingUsernamePassword realm (#32646) + +commit 318df2a107fb2e77a035bd18ce0a65cf54a88772 +Author: Jason Tedor +Date: Mon Aug 27 12:27:23 2018 -0400 + + Adjust BWC version on mapping version + + The introduction of mapping version on index metadata has been + backported to 6.x. This commit adjusts the BWC version around mapping + version to account for this backport. + +commit 5d9c2706085e6d68c98227fbe6df747892af3de6 +Author: Jay Modi +Date: Mon Aug 27 10:56:21 2018 -0600 + + Token API supports the client_credentials grant (#33106) + + This change adds support for the client credentials grant type to the + token api. The client credentials grant allows for a client to + authenticate with the authorization server and obtain a token to access + as itself. Per RFC 6749, a refresh token should not be included with + the access token and as such a refresh token is not issued when the + client credentials grant is used. + + The addition of the client credentials grant will allow users + authenticated with mechanisms such as kerberos or PKI to obtain a token + that can be used for subsequent access. + +commit 309fb2218197b0469ec27d10eebcaac18729e910 +Author: Jay Modi +Date: Mon Aug 27 10:26:25 2018 -0600 + + Build: forked compiler max memory matches jvmArgs (#33138) + + This commit removes the setting of the fork options maximum memory size + in our build plugin and instead adds the value in the gradle.properties + file to be alongside the value set in jvmArgs. + + This change is necessary when using parallel compilation as 512m is not + sufficient for parallel compilation on some machines. + +commit 2aef7e090048138316584b180c17ba86a2bf4702 +Author: Jason Tedor +Date: Mon Aug 27 12:21:11 2018 -0400 + + Introduce mapping version to index metadata (#33147) + + This commit introduces mapping version to index metadata. This value is + monotonically increasing and is updated on mapping updates. This will be + useful in cross-cluster replication so that we can request mapping + updates from the leader only when there is a mapping update as opposed + to the strategy we employ today which is to request a mapping update any + time there is an index metadata update. As index metadata updates can + occur for many reasons other than mapping updates, this leads to some + unnecessary requests and work in cross-cluster replication. + +commit 57835452228ba03238ef39e19829a67581689843 +Merge: 74312be0eaf 3d9ca4baee4 +Author: Tal Levy +Date: Mon Aug 27 08:19:05 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 3d9ca4baee4f537f6b40aa29729f171899b6ea00 +Author: Andrei Stefan +Date: Mon Aug 27 17:56:28 2018 +0300 + + SQL: Enable aggregations to create a separate bucket for missing values (#32832) + + Enable aggregations to create a separate bucket for missing values. + +commit c41c614527c70e54245df848ab17e23359837566 +Author: Jason Tedor +Date: Mon Aug 27 10:16:57 2018 -0400 + + Fix grammar in contributing docs + + This commit fixes an instance of odd comma placement in the contributing + docs. + +commit f7a9186372edcd9da31a89ac0565d53874db1839 +Author: Armin Braun +Date: Mon Aug 27 15:08:27 2018 +0200 + + SECURITY: Fix Compile Error in ReservedRealmTests (#33166) + + * This was broken by #32515 since the 5.x versions + were removed between PR creation and merge + +commit 1779d3376ac9c40d0916657c948555c2f043d5ab +Author: Shaunak Kashyap +Date: Mon Aug 27 08:42:40 2018 -0400 + + APM server monitoring (#32515) + + * Adding new MonitoredSystem for APM server + + * Teaching Monitoring template utils about APM server monitoring indices + + * Documenting new monitoring index for APM server + + * Adding monitoring index template for APM server + + * Copy pasta typo + + * Removing metrics.libbeat.config section from mapping + + * Adding built-in user and role for APM server user + + * Actually define the role :) + + * Adding missing import + + * Removing index template and system ID for apm server + + * Shortening line lengths + + * Updating expected number of built-in users in integration test + + * Removing "system" from role and user names + + * Rearranging users to make tests pass + +commit f1f6d4ed337a7b36895f344b964043e8c74fcd6b +Author: Mikita Karaliou +Date: Mon Aug 27 13:24:51 2018 +0300 + + Support only string `format` in date, root object & date range (#28117) + + Limit date `format` attribute to String values only. + + Closes #23650 + +commit e1e8cf382f629ae9576cd5b1d7a8add023e7dea6 +Author: Tanguy Leroux +Date: Mon Aug 27 09:18:26 2018 +0200 + + [Rollup] Move toBuilders() methods out of rollup config objects (#32585) + +commit 974f83909371920696dca4d9980c6e3f8c2c90bd +Author: Alpar Torok +Date: Mon Aug 27 08:47:42 2018 +0300 + + Fix forbiddenapis on java 11 (#33116) + + Cap forbiddenapis to java version 10 + +commit 30c3b363950e51e6d2ff96195fc1a54f0161c590 +Author: Alpar Torok +Date: Mon Aug 27 08:44:06 2018 +0300 + + Apply publishing to genreate pom (#33094) + +commit 47e9e72df23f3bb9a6983c8c4ac286385dab100c +Author: Martijn van Groningen +Date: Mon Aug 27 12:14:46 2018 +0700 + + reduce maximum number of writes to speed up test + +commit 06c0055c0f4756bf7b3db84c9e476e7a20a9758d +Author: Daniel Mitterdorfer +Date: Mon Aug 27 07:09:27 2018 +0200 + + Have circuit breaker succeed on unknown mem usage + + With this commit we implement a workaround for + https://bugs.openjdk.java.net/browse/JDK-8207200 which is a race + condition in the JVM that results in `IllegalArgumentException` to be + thrown in rare cases when we determine memory usage via `MemoryMXBean`. + As we do not want to fail requests in those cases we always return zero + memory usage. + + Relates #31767 + Relates #33125 + +commit 143cd9bbaa082fa9535990bba32c47739411cb4d +Author: Jason Tedor +Date: Sun Aug 26 15:57:52 2018 -0400 + + Do not lose default mapper on metadata updates (#33153) + + When applying index metadata updates we run through the mappings + updating them if needed. Today if there is not an update to the default + mapper, we can lose the default mapping. This means that, for example, + if we apply a settings update to an index we will lose the default + mapper. This happens because we were not guarding updating the default + mapping with a check that the default mapping was updated in the + metadata update. When there is no update in the metadata update, we need + to continue to preserve the previous default mapping. This commit + achieves this by moving the updating of the default mapping under the + same guard that we use for updating the default mapping source. We add a + test that fails before putting the update under a guard and now passes + after moving the update under the guard. + +commit f8b07a0d84eb5f838e2112d04b9241ba7d4597d3 +Author: Jason Tedor +Date: Sun Aug 26 09:36:17 2018 -0400 + + Fix a mappings update test (#33146) + + This commit fixes a mappings update test. The test is broken in the + sense that it passes, but for the wrong reason. The test here is testing + that if we make a mapping update but do not commit that mapping update + then the mapper service still maintains the previous document + mapper. This was not the case long, long ago when a mapping update would + update the in-memory state before the cluster state update was + committed. This test was passing, but it was passing because the mapping + update was never even updated. It was never even updated because it was + encountering a null pointer exception. Of course the in-memory state is + not going to be updated in that case, we are simply going to end up with + a failed cluster state update. Fixing that leads to another issue which + is that the mapping source does not even parse so again we would, of + course, end up with the in-memory state not being modified. We fix these + issues, assert that the result cluster state task completed + successfully, and finally that the in-memory state was not updated since + we never committed the resulting cluster state. + +commit fbe609d589fe57b1f822b2e5c77718af6e2faf53 +Author: Albert Zaharovits +Date: Sun Aug 26 14:49:32 2018 +0300 + + Reload Secure Settings REST specs & docs (#32990) + + This is a minimal REST API spec and docs for the REST handler + for the `_nodes/reload_secure_settings endpoint`. + + Relates #29135 + +commit c567ec4a0fee228be7d3c45c6be5c140cdb83db9 +Author: Albert Zaharovits +Date: Sun Aug 26 14:09:23 2018 +0300 + + Refactor CachingUsernamePassword realm (#32646) + + Refactors the logic of authentication and lookup caching in + `CachingUsernamePasswordRealm`. Nothing changed about + the single-inflight-request or positive caching. + +commit 75304f405b0e97f28c5e4b21017e47007328dbd9 +Merge: ef9607ea0cb 3376922e8b1 +Author: Nhat Nguyen +Date: Sat Aug 25 21:41:53 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add proxy support to RemoteClusterConnection (#33062) + TEST: Skip assertSeqNos for closed shards (#33130) + TEST: resync operation on replica should acquire shard permit (#33103) + Switch remaining x-pack tests to new style Requests (#33108) + Switch remaining tests to new style Requests (#33109) + Switch remaining ml tests to new style Requests (#33107) + Build: Line up IDE detection logic + Security index expands to a single replica (#33131) + HLRC: request/response homogeneity and JavaDoc improvements (#33133) + Checkstyle! + [Test] Fix sporadic failure in MembershipActionTests + Revert "Do NOT allow termvectors on nested fields (#32728)" + [Rollup] Move toAggCap() methods out of rollup config objects (#32583) + Fix race condition in scheduler engine test + +commit 3376922e8b1a587eb09c7f3d1016b0ee49cd4b9f +Author: Simon Willnauer +Date: Sat Aug 25 20:41:32 2018 +0200 + + Add proxy support to RemoteClusterConnection (#33062) + + This adds support for connecting to a remote cluster through + a tcp proxy. A remote cluster can configured with an additional + `search.remote.$clustername.proxy` setting. This proxy will be used + to connect to remote nodes for every node connection established. + We still try to sniff the remote clsuter and connect to nodes directly + through the proxy which has to support some kind of routing to these nodes. + Yet, this routing mechanism requires the handshake request to include some + kind of information where to route to which is not yet implemented. The effort + to use the hostname and an optional node attribute for routing is tracked + in #32517 + + Closes #31840 + +commit 9dad82ece8e13dd11b8301d41d4f0020c461050f +Author: Nhat Nguyen +Date: Fri Aug 24 21:02:13 2018 -0400 + + TEST: Skip assertSeqNos for closed shards (#33130) + + If a shard was closed, we return null for SeqNoStats. Therefore the + assertion assertSeqNos will hit NPE when it verifies a closed shard. + + This commit skips closed shards in assertSeqNos and enables this + assertion in AbstractDisruptionTestCase. + +commit 739a8d3d44e0f535491a53f13ab4a26a518dd73c +Author: Nhat Nguyen +Date: Fri Aug 24 20:25:13 2018 -0400 + + TEST: resync operation on replica should acquire shard permit (#33103) + + This change makes sure that resync operations on replicas in the test + framework are executed under shard permits as the production code. + +commit 1e9144d8e6358d1aff5fa377206c66520424022c +Author: Nik Everett +Date: Fri Aug 24 16:39:08 2018 -0400 + + Switch remaining x-pack tests to new style Requests (#33108) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/qa/saml-idp-tests` and + `x-pack/qa/security-setup-password-tests` projects to use the new + versions. + +commit dd4a8dc444f0fba20f486e1f9eed1bcb327e85b7 +Author: Nik Everett +Date: Fri Aug 24 16:37:25 2018 -0400 + + Switch remaining tests to new style Requests (#33109) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `client` and `distribution` projects to use + the new versions. + +commit 8bee6b3a922f5f8b86bdae125aab9171dd88a242 +Author: Nik Everett +Date: Fri Aug 24 16:36:40 2018 -0400 + + Switch remaining ml tests to new style Requests (#33107) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/plugin/ml/qa/native-multi-node-tests`, + `x-pack/plugin/ml/qa/single-node-tests` projects to use the new + versions. + +commit a9a66a09dc3d3ac373cf1dbd11f6befe6bdb42a8 +Author: Nik Everett +Date: Fri Aug 24 16:11:07 2018 -0400 + + Build: Line up IDE detection logic + + The IDE detection logic in build.gradle and settings.gradle has to match + or else Eclipse users will have a bad time. But in this case I think it + mostly just hits folks who use the Eclipse compiler server like me. + +commit 74312be0eafeac3d6cdc0311f452f8b84b30b4e5 +Merge: 52aa738d84b b86dad22ce2 +Author: Tal Levy +Date: Fri Aug 24 12:41:12 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit b86dad22ce2701a403d0a6073889141ab4578552 +Author: Jay Modi +Date: Fri Aug 24 12:51:22 2018 -0600 + + Security index expands to a single replica (#33131) + + This change removes the use of 0-all for auto expand replicas for the + security index. The use of 0-all causes some unexpected behavior with + certain allocation settings. This change allows us to avoid these with + a default install. If necessary, the number of replicas can be tuned by + the user. + + Closes #29933 + Closes #29712 + +commit ef9607ea0cbc2f928fd233e51ba67b392d7e81bd +Author: Jason Tedor +Date: Fri Aug 24 14:21:23 2018 -0400 + + Track fetch exceptions for shard follow tasks (#33047) + + This commit adds tracking and reporting for fetch exceptions. We track + fetch exceptions per fetch, keeping track of up to the maximum number of + concurrent fetches. With each failing fetch, we associate the from + sequence number with the exception that caused the fetch. We report + these in the CCR stats endpoint, and add some testing for this tracking. + +commit 52cf57ee2dbdeb2d6135ac7645b0bc8e956cea19 +Author: Benjamin Trent +Date: Fri Aug 24 13:18:50 2018 -0500 + + HLRC: request/response homogeneity and JavaDoc improvements (#33133) + +commit a023e64801547ce014d8ff672b24e21b1f441d49 +Author: Nik Everett +Date: Fri Aug 24 14:13:13 2018 -0400 + + Checkstyle! + + Catching your unused imports since 2001. + +commit 70030c18f1eb51c7b31db8657e162a5523462ac8 +Author: Jim Ferenczi +Date: Fri Aug 24 18:40:04 2018 +0200 + + [Test] Fix sporadic failure in MembershipActionTests + + Rewrite test that require Version.V_5 constants. + +commit 6f1ee76443c4d04095bce2c34a8cb474c5854e4a +Author: Mayya Sharipova +Date: Fri Aug 24 10:12:16 2018 -0400 + + Revert "Do NOT allow termvectors on nested fields (#32728)" + + This reverts commit fdff8f3db0093fa15cfa161f7dec80b715a48a43. + +commit 7fa8a728c4c7441a738e2e493912fbec80db5a56 +Author: Jason Tedor +Date: Fri Aug 24 09:48:54 2018 -0400 + + Make CCR QA tests build again (#33113) + + Welp, I broke this. I merged a change to auto-discover the CCR QA tests + by making :x-pack:plugin:ccr:check auto-discover the check tasks in the + qa sub-project. Yet, the check tasks for these sub-projects did not + depend on the necessary test tasks (as we were previously doing this + directly from the ccr build file. This commit fixes this! + +commit 7e5efad92918e95676fd047a2897cc626441661f +Author: Tanguy Leroux +Date: Fri Aug 24 15:31:41 2018 +0200 + + [Rollup] Move toAggCap() methods out of rollup config objects (#32583) + +commit 9fddf7e6a3ce13597d21c61d9818e301016b7b03 +Author: Jason Tedor +Date: Fri Aug 24 07:45:16 2018 -0400 + + Fix race condition in scheduler engine test + + This commit addresses a race condition in the scheduler engine test that + a listener that throws an exception does not cause other listeners to be + skipped. The race here is that we were counting down a latch, and then + throwing an exception yet an assertion that expected the exception to + have been thrown already could execute after the latch was counted down + for the final time but before the exception was thrown and acted upon by + the scheduler engine. This commit addresses this by moving the counting + down of the latch to definitely be after the exception was acted upon by + the scheduler engine. + +commit 91a052b617a53e63698f9eb084b4716e4ab24903 +Merge: b0f22d67c46 619e0b28b97 +Author: Jason Tedor +Date: Fri Aug 24 06:55:10 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add hook to skip asserting x-content equivalence (#33114) + Muted testListenersThrowingExceptionsDoNotCauseOtherListenersToBeSkipped + [Rollup] Move getMetadata() methods out of rollup config objects (#32579) + Muted testEmptyAuthorizedIndicesSearchForAllDisallowNoIndices + Update Google Cloud Storage Library for Java (#32940) + Remove unsupported Version.V_5_* (#32937) + +commit 619e0b28b97b8ea7cddf6b29aaa1a2756da9b363 +Author: Jason Tedor +Date: Fri Aug 24 06:53:44 2018 -0400 + + Add hook to skip asserting x-content equivalence (#33114) + + This commit adds a hook to AbstractSerializingTestCase to enable + skipping asserting that the x-content of the test instance and an + instance parsed from the x-content of the test instance are the + same. While we usually expect these to be the same, they will not be the + same when exceptions are involved because the x-content there is lossy. + +commit a2f0a1a0cbd5025dfb125a24cb68c38413e3ab2b +Merge: 1d8745036f1 879a90b9992 +Author: Andrei Stefan +Date: Fri Aug 24 13:14:37 2018 +0300 + + Merge branch 'master' of https://github.com/elastic/elasticsearch + +commit 1d8745036f19786f62808cc78a5dedbf2cfcda21 +Author: Andrei Stefan +Date: Fri Aug 24 13:14:03 2018 +0300 + + Muted testListenersThrowingExceptionsDoNotCauseOtherListenersToBeSkipped + +commit 879a90b99922f29c8764ce5de7d4349647cd56b7 +Author: Tanguy Leroux +Date: Fri Aug 24 11:57:46 2018 +0200 + + [Rollup] Move getMetadata() methods out of rollup config objects (#32579) + + This committ removes the getMetadata() methods from the DateHistoGroupConfig + and HistoGroupConfig objects. This way the configuration objects do not rely on RollupField.formatMetaField() anymore and do not expose a getMetadata() + method that is tighlty coupled to the rollup indexer. + +commit b0f22d67c46b7882ac4c53e842b3206e9c5a0e86 +Author: Martijn van Groningen +Date: Fri Aug 24 16:56:29 2018 +0700 + + fixed not returning response instance + +commit 66e458b78b547d68ed74c4ddae4924b4fdb0d0b2 +Author: Andrei Stefan +Date: Fri Aug 24 12:36:23 2018 +0300 + + Muted testEmptyAuthorizedIndicesSearchForAllDisallowNoIndices + +commit c5e5a97a343d970f796b5d5748408ff765127d8f +Author: Tanguy Leroux +Date: Fri Aug 24 10:55:23 2018 +0200 + + Update Google Cloud Storage Library for Java (#32940) + + This commit updated the google-cloud-storage library from version 1.28.0 + to version 1.40.0. + +commit f4e9729d647add3d42b5bdd091278dcb3af727af +Author: Jim Ferenczi +Date: Fri Aug 24 09:51:21 2018 +0200 + + Remove unsupported Version.V_5_* (#32937) + + This change removes the es 5x version constants and their usages. + +commit 575f33941cae81e1d4b49f3424eb8f2e550bb9cb +Author: Martijn van Groningen +Date: Fri Aug 24 12:51:26 2018 +0700 + + Required changes after merging in master branch. + +commit 82592dda5a1bd3f3b2f3c955297d52724f621ed6 +Merge: 9623cf6cde3 a211d24bda9 +Author: Martijn van Groningen +Date: Fri Aug 24 12:22:11 2018 +0700 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: (62 commits) + [DOCS] Add docs for Application Privileges (#32635) + Add versions 5.6.12 and 6.4.1 + Do NOT allow termvectors on nested fields (#32728) + [Rollup] Return empty response when aggs are missing (#32796) + [TEST] Add some ACL yaml tests for Rollup (#33035) + Move non duplicated actions back into xpack core (#32952) + Test fix - GraphExploreResponseTests should not randomise array elements Closes #33086 + Use `addIfAbsent` instead of checking if an element is contained + TESTS: Fix Random Fail in MockTcpTransportTests (#33061) + HLRC: Fix Compile Error From Missing Throws (#33083) + [DOCS] Remove reload password from docs cf. #32889 + HLRC: Add ML Get Buckets API (#33056) + Watcher: Improve error messages for CronEvalTool (#32800) + Search: Support of wildcard on docvalue_fields (#32980) + Change query field expansion (#33020) + INGEST: Cleanup Redundant Put Method (#33034) + SQL: skip uppercasing/lowercasing function tests for AZ locales as well (#32910) + Fix the default pom file name (#33063) + Switch ml basic tests to new style Requests (#32483) + Switch some watcher tests to new style Requests (#33044) + ... + +commit a211d24bda99b810868a79215d4d053332600cd4 +Author: Tim Vernum +Date: Fri Aug 24 11:04:02 2018 +1000 + + [DOCS] Add docs for Application Privileges (#32635) + +commit 52aa738d84b86cb0f792bf5c03105f249160698a +Author: Lee Hinman +Date: Thu Aug 23 15:37:02 2018 -0600 + + Remove canSetPolicy, canUpdatePolicy, and canRemovePolicy (#33037) + + * Remove canSetPolicy, canUpdatePolicy and canRemovePolicy + + Since we now store a pre-compiled list of steps for an index's phase in the + `PolicyStepsRegistry`, we no longer need to worry about updating policies as any + updates won't affect the current phase, and will only be picked up on phase + transitions. + + This also removes the tests that test these methods + + Relates to #29823 + +commit 935b28087b24fa8c3afc34332eed4a739c29763b +Merge: 650f12af1e4 8f16696fe12 +Author: Gordon Brown +Date: Thu Aug 23 14:56:25 2018 -0600 + + Merge branch 'master' into index-lifecycle + +commit 8f16696fe12bd406327b0f715f513a33c642ec37 +Author: Michael Basnight +Date: Thu Aug 23 15:45:25 2018 -0500 + + Add versions 5.6.12 and 6.4.1 + +commit fdff8f3db0093fa15cfa161f7dec80b715a48a43 +Author: Mayya Sharipova +Date: Thu Aug 23 16:46:47 2018 -0400 + + Do NOT allow termvectors on nested fields (#32728) + + Requesting _termvectors on a nested field or any sub-fields of a nested field + returns empty results. + + Closes #21625 + +commit 8f8d3a5556abaaabbfbfd588fb96d908cc330ff9 +Author: Zachary Tong +Date: Thu Aug 23 16:15:37 2018 -0400 + + [Rollup] Return empty response when aggs are missing (#32796) + + If a search request doesn't contain aggs (or an empty agg object), + we should just retun an empty response. This is how the normal search + API works if you specify zero hits and empty aggs. + + The existing behavior throws an exception because it tries to send + an empty msearch. + + Closes #32256 + +commit 650f12af1e414aa1c5356a2b61abf2407aa4cfbd +Author: Gordon Brown +Date: Thu Aug 23 13:50:15 2018 -0600 + + Duplicate Protocol classes into Core + + This is needed as with recent changes to master (see #32952), protocol + is no longer accessible from core, so these classes need to be + duplicated in both places. + +commit 191bd7c03162f31f88404c08958a8010eb1cff78 +Author: Gordon Brown +Date: Thu Aug 23 12:08:16 2018 -0600 + + Fix Gradle configuration + + This change was made to master, this commit brings it over to + index-lifecycle. + + See #32409 + +commit 1f13c77b49f46f13b7a2dd40581d942f5ccedb05 +Merge: fd88ab8c75e 0da981a6a9d +Author: Gordon Brown +Date: Thu Aug 23 11:52:59 2018 -0600 + + Merge branch 'master' into index-lifecycle + +commit a0d32f59475070756de43178ccaa5558173959f2 +Author: Yannick Welsch +Date: Thu Aug 23 19:18:52 2018 +0200 + + Zen2: Add leader-side join handling logic (#33013) + + Adds the logic for handling joins by a prospective leader. Introduces the Coordinator class with the + basic lifecycle modes (candidate, leader, follower) as well as a JoinHelper class that contains most + of the plumbing for handling joins. + +commit 0da981a6a9d3c687dbff6de6f866997b5a8c30d2 +Author: Zachary Tong +Date: Thu Aug 23 11:43:48 2018 -0400 + + [TEST] Add some ACL yaml tests for Rollup (#33035) + + These two tests compliment the existing unit tests which check Rollup's + ACL/security integration. + + The first test creates to indices, puts a document in each one, and then + assigns a role to the test user that can only access one of the indices. + A rollup job is created with a pattern that would match both indices, + and we verify that only the allowed document was rolled up (e.g. verifying + that the unpermissioned index stays hidden). + + The second test creates a single index with two documents tagged by + the keyword "public"/"private". An attribute-based role is created + that only allows viewing "public" documents. We then verify the rollup + job only rolled the "public" doc, and not the "private" one. + +commit fd88ab8c75e580f76e5671cd89251e1147177656 +Author: Colin Goodheart-Smithe +Date: Thu Aug 23 16:36:57 2018 +0100 + + Fixes shrink action to remove single ndoe allocation (#33091) + + This change fixes the shrink action so when the shrink is performed we + remove the single node allocation fromt eh shard allocation filtering + settings. Without this fix replicas cannot be allocated after we have + performed the shrink and we cannot make progress with the rest of the + shink aciton. + + This change also fixes a bug in the explain API where the maste node + timeout was being set to null if it wasn't provided instead of using + its default value causing a NPE + +commit 644c0de5ec0e14ca6ab3abc8d4a18b96f95a023b +Author: Michael Basnight +Date: Thu Aug 23 09:48:53 2018 -0500 + + Move non duplicated actions back into xpack core (#32952) + + Most actions' request and response were moved from xpack core into + protocol. We have decided to instead duplicate the actions in the HLRC + instead of trying to reuse them. This commit moves the non duplicated + actions back into xpack core and severs the tie between xpack core and + protocol so no other actions can be moved and not duplicated. + +commit f860e589a629bd89f24f022a50c8a2fcb7a77673 +Author: markharwood +Date: Thu Aug 23 15:00:30 2018 +0100 + + Test fix - GraphExploreResponseTests should not randomise array elements + Closes #33086 + +commit f3cfd4504f4828db2f2b694fd0bba5654121cbfc +Author: Simon Willnauer +Date: Thu Aug 23 13:33:39 2018 +0200 + + Use `addIfAbsent` instead of checking if an element is contained + + Relates to #32988 + +commit 917e5a8c949824e653d92b443768e3909c375992 +Author: Armin Braun +Date: Thu Aug 23 13:19:21 2018 +0200 + + TESTS: Fix Random Fail in MockTcpTransportTests (#33061) + + * `foobar.txGet()` appears to return before `serviceB.stop()` returns, causing `ServiceB.close()` to run concurrently with the `stop` call and running into a race codition + * Closes #32863 + +commit 61f5c188e0a577555db382dd0fe9a4222da9df1a +Author: Armin Braun +Date: Thu Aug 23 12:24:32 2018 +0200 + + HLRC: Fix Compile Error From Missing Throws (#33083) + + * 50441f97ae745814db96c262e99d0f465aca5b2c#diff-53a95fe7ded21313483f1b2f15977395L72 removed the throws breaking compilation here + +commit 07cce953059e6e0001366405d555187c8498f7e3 +Author: Albert Zaharovits +Date: Thu Aug 23 12:08:06 2018 +0300 + + [DOCS] Remove reload password from docs cf. #32889 + + Reload call `_nodes/reload_secure_settings` is not requiring + an empty password anymore (#32889). Reflect this in docs. + +commit 50441f97ae745814db96c262e99d0f465aca5b2c +Author: Dimitris Athanasiou +Date: Thu Aug 23 09:35:06 2018 +0100 + + HLRC: Add ML Get Buckets API (#33056) + + Relates #29827 + +commit f84ed14294db30ec2fb068438f5a9082ef0302f1 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Thu Aug 23 10:07:59 2018 +0200 + + Watcher: Improve error messages for CronEvalTool (#32800) + + CronEvalTool prints an error only for cron expressions that result in + no upcoming time events. + + If a cron expression results in less than the specified count + (default 10) time events, now all the coming times are printed + without displaying error message. + + Closes #32735 + +commit d7219c05a27ebdee7f0a56ed52d5f625bfb11f5a +Author: Ignacio Vera +Date: Thu Aug 23 10:04:00 2018 +0200 + + Search: Support of wildcard on docvalue_fields (#32980) + + * Search: Support of wildcard on docvalue_fields + + For consistency with stored_fields, docvalue_fields should support the use of wildcards. + Documentation of doc values fields is updated accordingly. + + See also: #26390 + + Closes #26299 + +commit ffe895e16e38917f88789443ed975ad1fc35056c +Author: Jim Ferenczi +Date: Thu Aug 23 09:52:48 2018 +0200 + + Change query field expansion (#33020) + + This commit changes the query field expansion for query parsers + to not rely on an hardcoded list of field types. Instead we rely on + the type of exception that is thrown by MappedFieldType#termQuery to + include/exclude an expanded field. + + Supersedes #31655 + + Closes #31798 + +commit 46247ff1f9788ca46958833fbb15e7afcc8f6f4e +Author: Armin Braun +Date: Thu Aug 23 07:43:36 2018 +0200 + + INGEST: Cleanup Redundant Put Method (#33034) + +commit de95dead2dae32ffcdf7cc22e1c37ea4896d1f5a +Author: Andrei Stefan +Date: Thu Aug 23 00:21:38 2018 +0300 + + SQL: skip uppercasing/lowercasing function tests for AZ locales as well (#32910) + + * Added the rest of the Locales that have different behavior for uppercasing/lowercasing scenarios to the skip list + +commit edd477a15e07f021f40e496379df09c4e2e792c1 +Author: Alpar Torok +Date: Wed Aug 22 21:53:42 2018 +0300 + + Fix the default pom file name (#33063) + + Before this change the default was fixed at compile time and not picking + up changes in the build script. + +commit dfc70ddcc77a33e7bd6f1384383023088b92ad47 +Author: Tal Levy +Date: Wed Aug 22 11:41:09 2018 -0700 + + rename pre-phase/pre-action to new/init (#32996) + + this will keep things more consistent with the initial PhaseAfterStep, which has a phase name of `new` + +commit 0cc99d270c6ad290efd600444cfa4dacda79c57d +Author: Nik Everett +Date: Wed Aug 22 14:23:43 2018 -0400 + + Switch ml basic tests to new style Requests (#32483) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/qa/ml-basic-multi-node` project to use + the new versions. + +commit c3438bc8d87ddb1786646505d84880b2018623c5 +Author: Nik Everett +Date: Wed Aug 22 14:02:39 2018 -0400 + + Switch some watcher tests to new style Requests (#33044) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/qa/smoke-test-monitoring-with-watcher`, + `x-pack/qa/smoke-test-watcher`, and + `x-pack/qa/smoke-test-watcher-with-security` projects to use the new + versions. + +commit 393eec148217a721281b111ea8ab95c436ec5684 +Author: Luca Cavanna +Date: Wed Aug 22 17:23:54 2018 +0200 + + Set maxScore for empty TopDocs to Nan rather than 0 (#32938) + + We used to set `maxScore` to `0` within `TopDocs` in situations where there is really no score as the size was set to `0` and scores were not even tracked. In such scenarios, `Float.Nan` is more appropriate, which gets converted to `max_score: null` on the REST layer. That's also more consistent with lucene which set `maxScore` to `Float.Nan` when merging empty `TopDocs` (see `TopDocs#merge`). + +commit abb4c183f1e9ed8835f4c4bfede6eebc9c85c3ef +Author: Dimitrios Liappis +Date: Wed Aug 22 18:18:30 2018 +0300 + + Clarify ignore_above behavior with arrays of strings + + Currently docs don't explain how `ignore_above` behaves with arrays of + strings. + + Clarify how `ignore_above` applies for arrays of strings and + also note that all string(s) will still be visible in the + `_source` field. + + Relates #33057 + +commit 528e727999bc5bf851019e0848d1c2bbb8c17f34 +Author: Jason Tedor +Date: Wed Aug 22 10:19:30 2018 -0400 + + Fix method reference in comment in SchedulerEngine + + This commit fixes the name of a method reference in a comment in + SchedulerEngine. + +commit e9912081dd3507bf3ab1ceabd949ca692a5a92eb +Author: Michael Basnight +Date: Wed Aug 22 09:19:58 2018 -0500 + + HLRC: Create server agnostic request and response (#32912) + + The HLRC has historically reused the same Request and Response classes + that the server module uses. This commit deprecates the use of any + server module Request and Response classes, and adds a small bit of + validation logic that differs from server slightly, in that it does not + assume a check for a null ValidationException class is not enough to + determine if validation failed. + +commit 67bfb765ee8b7958a4d7a26b17f653ae8955878d +Author: Jason Tedor +Date: Wed Aug 22 10:18:07 2018 -0400 + + Refactor Netty4Utils#maybeDie (#33021) + + In our Netty layer we have had to take extra precautions against Netty + catching throwables which prevents them from reaching the uncaught + exception handler. This code has taken on additional uses in NIO layer + and now in the scheduler engine because there are other components in + stack traces that could catch throwables and suppress them from reaching + the uncaught exception handler. This commit is a simple cleanup of the + iterative evolution of this code to refactor all uses into a single + method in ExceptionsHelper. + +commit ead198bf2e76d907ed218b5aaddc60d53b7add72 +Author: Simon Willnauer +Date: Wed Aug 22 14:13:27 2018 +0200 + + Add settings updater for 2 affix settings (#33050) + + Today we can only have non-affix settings updated and consumed _together_. + Yet, there are use-cases where two affix settings depend on each other which + makes using the hard without consuming updates together. Unfortunately, there is + not straight forward way to have N settings updated together in a type-safe way + having 2 still serves a large portion of use-cases. + +commit 262d3c0783d771ea6b196ae2d9e38f44ec33d89c +Author: Nhat Nguyen +Date: Wed Aug 22 07:57:44 2018 -0400 + + Allow engine to recover from translog upto a seqno (#33032) + + This change allows an engine to recover from its local translog up to + the given seqno. The extended API can be used in these use cases: + + When a replica starts following a new primary, it resets its index to + the safe commit, then replays its local translog up to the current + global checkpoint (see #32867). + + When a replica starts a peer-recovery, it can initialize the + start_sequence_number to the persisted global checkpoint instead of the + local checkpoint of the safe commit. A replica will then replay its + local translog up to that global checkpoint before accepting remote + translog from the primary. This change will increase the chance of + operation-based recovery. I will make this in a follow-up. + + Relates #32867 + +commit b02150a5ed8bd4f2dc393bfd3b6de0518149c468 +Author: Benjamin Trent +Date: Wed Aug 22 06:54:08 2018 -0500 + + HLRC: close job refactor (#33031) + + * HLRC: close job refactor + + * Changing refactor to make job_id a string + + * Changing set entity methodology + +commit 43f6f435f5b28e7a65c24d85fec0690c1d2e58b5 +Author: Sergey +Date: Wed Aug 22 12:27:37 2018 +0300 + + [DOCS] Update remote-info.asciidoc (#32978) + +commit ab000323fa7be57858c5470516877c69d00f058f +Author: David Turner +Date: Wed Aug 22 09:09:08 2018 +0100 + + Allow extension of CapturingTransport by subclasses (#33012) + + Today, CapturingTransport#createCapturingTransportService creates a transport + service with a connection manager with reasonable default behaviours, but + overriding this behaviour in a consumer is a litle tricky. Additionally, the + default behaviour for opening a connection duplicates the content of the + CapturingTransport#openConnection() method. + + This change removes this duplication by delegating to openConnection() and + introduces overridable nodeConnected() and onSendRequest() methods so that + consumers can alter this behaviour more easily. + + Relates #32246 in which we test the mechanisms for opening connections to + unknown (and possibly unreachable) nodes. + +commit ffb1a5d5b795e40fb7b9afff2a04fb0fc3166d5c +Author: Simon Willnauer +Date: Wed Aug 22 08:45:08 2018 +0200 + + Expose `max_concurrent_shard_requests` in `_msearch` (#33016) + + Today `_msearch` doesn't allow modifying the `max_concurrent_shard_requests` + per sub search request. This change adds support for setting this parameter on + all sub-search requests in an `_msearch`. + + Relates to #31877 + +commit 82d10b484a20514e70f6d773544ffea62e57f711 +Author: Alpar Torok +Date: Wed Aug 22 09:05:22 2018 +0300 + + Run forbidden api checks with runtimeJavaVersion (#32947) + + Run forbidden APIs checks with runtime hava version + +commit 0a4b55c9c0f1818d2199cc8468b70b9cc98e5ae8 +Author: Ioannis Kakavas +Date: Wed Aug 22 08:37:50 2018 +0300 + + [DOCS] Add RequestedAuthnContext Documentation (#32946) + + Add documentation for #31238 + + - Add documentation for the req_authn_context_class_ref setting + - Add a section in SAML Guide regarding the use of SAML + Authentication Context. + +commit 9f588c953f514a3fbdef0e4d675614203ef1e8ee +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Aug 22 15:23:39 2018 +1000 + + [TEST] Split tests and skip file permission test on Windows (#32781) + + Changes to split tests for keytab file test cases instead of + randomized testing for testing branches in the code in the + same test. + On windows platform, for keytab file permission test, we + required additional security permissions for the test + framework. As this was the only test that required those + permissions, skipping that test on windows platform. + The same scenario gets tested in *nix environments. + + Closes#32768 + +commit e2ea83d2174ad43027bf8756e493e36d4c8a738c +Author: Benjamin Trent +Date: Tue Aug 21 21:02:28 2018 -0500 + + HLRC: Add ML Get Job (#32960) + + * HLRC: Adding GET ML Job info API + + * HLRC: Adding GET Job ML API + + * Fixing QueryPage license header + + * Adding serialization tests, addressing minor issues + + * Renaming querypage, changing the dependency on it + + * Making things immutable + + * Fixing build failure due to method rename + +commit 07b3ff9fe711c076e12ac066472f6a624ec78bd4 +Author: Russ Cam +Date: Wed Aug 22 11:26:53 2018 +1000 + + Add beta label to MSI on install Elasticsearch page (#28126) + + The main installation instructions page for the Windows MSI installer includes a header at the top to indicate that the installer is in beta, but the Installing Elasticsearch page does not. This commit adds the beta label to the MSI entry within the installation options. + +commit 55cb08a352ddccfaf43c9dd5aae5225bbe6dc2a7 +Author: Tal Levy +Date: Tue Aug 21 17:59:32 2018 -0700 + + move ESLoggerFactory usage to LogManager (#33043) + + Work done in #32513 has deprecated the old constructor in favor of + log4j2's LogManager + +commit 2c81d7f77ec90c94446be3a8fe15d461a6f2d362 +Author: Nik Everett +Date: Tue Aug 21 20:03:28 2018 -0400 + + Build: Rework shadow plugin configuration (#32409) + + This reworks how we configure the `shadow` plugin in the build. The major + change is that we no longer bundle dependencies in the `compile` configuration, + instead we bundle dependencies in the new `bundle` configuration. This feels + more right because it is a little more "opt in" rather than "opt out" and the + name of the `bundle` configuration is a little more obvious. + + As an neat side effect of this, the `runtimeElements` configuration used when + one project depends on another now contains exactly the dependencies needed + to run the project so you no longer need to reference projects that use the + shadow plugin like this: + + ``` + testCompile project(path: ':client:rest-high-level', configuration: 'shadow') + ``` + + You can instead use the much more normal: + + ``` + testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}" + ``` + +commit 67b5a83a9ac1e28141ceee390e4aabe601192b14 +Author: Julie Tibshirani +Date: Tue Aug 21 16:33:42 2018 -0700 + + Ensure that _exists queries on keyword fields use norms when they're available. (#33006) + +commit 767c69593c67befb843686de8ea51b7bc87728c9 +Author: Jim Ferenczi +Date: Tue Aug 21 22:15:09 2018 +0200 + + Fix quoted _exists_ query (#33019) + + This change in the `query_string` query fixes the detection of the special + `_exists_` field when it is used with a quoted term. + + Closes #28922 + +commit 15727ae8ed6e9a5abf03009b15e1ec93c051fa50 +Author: debadair +Date: Tue Aug 21 13:13:07 2018 -0700 + + [DOCS] Fixed formatting of Example headings. (#33038) + +commit 8b43e21521b4d0f3409439d428eda156d8744cf1 +Author: Jim Ferenczi +Date: Tue Aug 21 22:12:53 2018 +0200 + + Fix multi fields empty query (#33017) + + This change fixes empty query removal when all fields remove the search term in + `simple_query_string`, `multi_match` and `query_string`. + + Closes #33009 + +commit f311680176bd68d74308bd5173d9703a8c45f263 +Author: Nik Everett +Date: Tue Aug 21 16:04:16 2018 -0400 + + Monitoring: Clean up MonitoringIT + + We recently reenabled MonitoringIT to hunt down #29880 but some of its + assertions were out of date. This updates the assertions. + +commit 96869b253c7cb4adf22f5ccb22798e331e8d040f +Author: Tal Levy +Date: Tue Aug 21 12:29:41 2018 -0700 + + conditionally update CS only if StepInfo changes (#33004) + + If we are waiting on a condition to be met, and the reason + it is not completed is unchanged, we find ourselves updating + cluster state over and over again and kicking of the ILM listeners + to re-check. This is overkill and can generate way too many + cluster state updates + +commit 6780ab9d5c6fd08ff603f908f3fac921b6c31ef8 +Author: Tal Levy +Date: Tue Aug 21 12:27:53 2018 -0700 + + add user authentication test for ILM (#32826) + +commit 3973bb4028cd0536eef0f178fb66f984d3a74246 +Author: Igor Motov +Date: Tue Aug 21 14:59:37 2018 -0400 + + Fix north pole overflow error in GeoHashUtils.bbox() (#32891) + + Fixes an overflow error in GeoHashUtils.bbox() calculation of a + bounding box for geohashes with maximum precision located next to the + north pole. + +commit fcf8cadd9a5114da8b8536338d058289daa88d99 +Author: Nik Everett +Date: Tue Aug 21 14:48:53 2018 -0400 + + Switch some x-pack tests to new style Requests (#32500) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/qa/audit-tests`, + `x-pack/qa/ml-disabled`, and `x-pack/qa/multi-node` projects to use the + new versions. + +commit 28a0df2c7fbaa46fa3dd5df671bdc6fa16fd3711 +Author: Dimitris Athanasiou +Date: Tue Aug 21 18:12:28 2018 +0100 + + HLRC: Clear ML data after client tests (#33023) + + This commit duplicates the `MlRestTestStateCleaner` to make sure + all ML data is removed after each test. After implementing + the job and datafeed APIs in the HLRC, we shall replace this + implementation with one using the HLRC itself. + + Closes #32993 + +commit 9623cf6cde3fa6c015933213db16a209b2351ab2 +Author: Jason Tedor +Date: Tue Aug 21 12:51:55 2018 -0400 + + Find CCR QA sub-projects automatically (#33027) + + Today we are by-hand maintaining a list of CCR QA sub-projects that the + check task depends on. This commit simplifies this by finding these + sub-projects automatically and adding their check task as dependencies + of the CCR check task. + +commit 28d12b05b77472d0596df319d7129cea3ba9154a +Author: Jason Tedor +Date: Tue Aug 21 12:23:21 2018 -0400 + + Move ML tests to be sub-projects of ML (#33026) + + This commit moves the ML QA tests to be a sub-project of ML. The purpose + of this refactoring is to enable ML developers to run + :x-pack:plugin:ml:check and run the vast majority of a ML tests with a + single command (this still does not contain the ML REST tests, nor the + upgrade tests). This simplifies local development for faster iteration. + +commit bdfcc326d7ee351390fb26f928261926c39ae6ff +Author: Jason Tedor +Date: Tue Aug 21 11:02:25 2018 -0400 + + Enable avoiding mmap bootstrap check (#32421) + + The maximum map count boostrap check can be a hindrance to users that do + not own the underlying platform on which they are executing + Elasticsearch. This is because addressing it requires tuning the kernel + and a platform provider might now allow this, especially on shared + infrastructure. However, this bootstrap check is not needed if mmapfs is + not in use. Today we do not have a way for the user to communicate that + they are not going to use mmapfs. This commit therefore adds a setting + that enables the user to disallow mmapfs. When mmapfs is disallowed, the + maximum map count bootstrap check is not enforced. Additionally, we + fallback to a different default index store and prevent the explicit use + of mmapfs for an index. + +commit 5b446b81efd500fcfef424d871a972c12e853ece +Author: Adrien Grand +Date: Tue Aug 21 16:53:59 2018 +0200 + + Reenable MonitoringIT#testMonitoringService. + + When discussing this test, it made little sense that testMonitoringService + would fail but not testMonitoringBulk given their similarity. So we argeed to + enable it again. + + Relates #29880 + +commit 75cae4560c146e12bc6739a640d7becd7812978e +Author: Colin Goodheart-Smithe +Date: Tue Aug 21 14:47:54 2018 +0100 + + fix compile error after merge + +commit 1b583978e9659de9ba6506ee6c4bbadff174ef65 +Author: Ioannis Kakavas +Date: Tue Aug 21 16:20:00 2018 +0300 + + [DOCS] Add FIPS 140-2 documentation (#32928) + + * Add relevant documentation for FIPS 140-2 compliance. + * Introduce `fips_mode` setting. + * Discuss necessary configuration for FIPS 140-2 + * Discuss introduced limitations by FIPS 140-2 + +commit 3f91bbfa6b408c2e72471c39cdeef4cbfe30eef2 +Author: Benjamin Trent +Date: Tue Aug 21 08:08:26 2018 -0500 + + [ML] Allowing _close to accept body payloads for options (#32989) (#33000) + +commit 38bdf9ce3238c5399e18b33efba34df3a269e17c +Author: markharwood +Date: Tue Aug 21 13:29:18 2018 +0100 + + HLRC GraphClient and associated tests (#32366) + + GraphClient for the high level REST client and associated tests. + Part of #29827 work + +commit 10c60fae93fc82469d3dcc9f330035e2a5818bb7 +Merge: 81c9e8d939e 65d4f278738 +Author: Colin Goodheart-Smithe +Date: Tue Aug 21 11:54:06 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 81c9e8d939e7a28f8d547ad074026430575fc058 +Author: Colin Goodheart-Smithe +Date: Tue Aug 21 11:50:40 2018 +0100 + + make operation mode YAML test more robust + +commit 92076497e507ab27bbb603f8dbcc6c2e236eaafa +Author: Simon Willnauer +Date: Tue Aug 21 12:43:25 2018 +0200 + + Use a dedicated ConnectionManger for RemoteClusterConnection (#32988) + + + This change introduces a dedicated ConnectionManager for every RemoteClusterConnection + such that there is not state shared with the TransportService internal ConnectionManager. + All connections to a remote cluster are isolated from the TransportService but still uses + the TransportService and it's internal properties like the Transport, tracing and internal + listener actions on disconnects etc. + This allows a remote cluster connection to have a different lifecycle than a local cluster connection, + also local discovery code doesn't get notified if there is a disconnect on from a remote cluster and + each connection can use it's own dedicated connection profile which allows to have a reduced set of + connections per cluster without conflicting with the local cluster. + + Closes #31835 + +commit 65d4f2787382a028653df2ffdfb25448ef082a77 +Author: Ioannis Kakavas +Date: Tue Aug 21 12:05:42 2018 +0300 + + [DOCS] Add configurable password hashing docs (#32849) + + * [DOCS] Add configurable password hashing docs + + Adds documentation about the newly introduced configuration option + for setting the password hashing algorithm to be used for the users + cache and for storing credentials for the native and file realm. + +commit 200078734c5b603907d2ba45d5e001ed787f0506 +Author: Armin Braun +Date: Tue Aug 21 10:13:32 2018 +0200 + + INGEST: Simplify IngestService (#33008) + + * INGEST: Simplify IngestService + + * Follow up to #32617 + * Flatten redundant inner classes of `IngestService` + +commit b595b1a20c895fac3c1e9a6d0e65c7f02f6c03df +Author: Tim Vernum +Date: Tue Aug 21 17:58:37 2018 +1000 + + Handle 6.4.0+ BWC for Application Privileges (#32929) + + When the application privileges feature was backported to 6.x/6.4 the + BWC version checks on the backport were updated to 6.4.0, but master + was not updated. + + This commit updates all relevant version checks, and adds tests. + +commit e4ef12798e5689f133bea50630813912397e00f2 +Author: David Turner +Date: Tue Aug 21 08:04:30 2018 +0100 + + Add PeerFinder#onFoundPeersUpdated (#32939) + + Today the PeerFinder silently updates the set of found peers as new peers are + discovered and old ones are disconnected, and elections are scheduled + independently of these changes. In fact, it would be better if the election + scheduler were only activated on discovery of a quorum of peers. This commit + introduces the `onFoundPeersUpdated` method that allows this flow. + +commit 38886e8f23153f69f6d4801dda121cfa352775f9 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Aug 21 16:30:28 2018 +1000 + + [DOCS] Add Kerberos troubleshooting documentation (#32803) + + This commit adds troubleshooting section for Kerberos. + Most of the times the problems seen are caused due to invalid + configurations like keytab missing principals or credentials + not up to date. Time synchronization is an important part for + Kerberos infrastructure and the time skew can cause problems. + To debug further documentation explains how to enable JAAS + Kerberos login module debugging and Kerberos/SPNEGO debugging + by setting JVM system properties. + +commit b08d02e3b73851b8d4d8446b13c521da6e06b4f0 +Author: Jason Tedor +Date: Mon Aug 20 23:33:18 2018 -0400 + + Implement CCR licensing (#33002) + + This commit implements licensing for CCR. CCR will require a platinum + license, and administrative endpoints will be disabled when a license is + non-compliant. + +commit 8fc213f237ccffc12870a7a0d845c91d8ecbab3b +Author: Armin Braun +Date: Tue Aug 21 05:05:32 2018 +0200 + + INGEST: Move all Pipeline State into IngestService (#32617) + + * INGEST: Move all Pipeline State into IngestService + + * Moves all pipeline state into the ingest service + * Retains the existing pipeline store and pipeline execution service as inner classes to make the review easier, they should be flattened out in the next step + * All tests for these classes were copied (and adapted) to the ingest service tests + * This is a refactoring step to enable a clean implementation of a pipeline processor (See #32473) + +commit 6d62d6755aeded636d7c52bda7e0331a9aaa9f1e +Author: Jason Tedor +Date: Mon Aug 20 22:58:07 2018 -0400 + + Fix typo in comment in scheduler engine + + This commit fixes a minor typo in a big block comment in + SchedulerEngine.java. + +commit ad0a965db9af15c49618a2f65c847ad4bbae1547 +Author: Jason Tedor +Date: Mon Aug 20 22:07:16 2018 -0400 + + Protect scheduler engine against throwing listeners (#32998) + + There are two problems with the scheduler engine today. Both relate to + listeners that throw. + + The first problem is that any triggered listener that throws a plain old + exception will cause no additional listeners to be triggered for the + event, and will also cause the scheduler to never be invoked again. This + leads to lost events and is bad. + + The second problem is that any triggered listener that throws an error + of the fatal kind will not lead to that error because caught by the + uncaught exception handler. This is because the triggered listener is + executed as a future task under a scheduled thread pool executor. A + throwable there goes caught by the JDK framework and set as the outcome + on the future task. Since we never inspect these tasks for their + outcomes, nor is there a good place to do this, we have to handle these + errors ourselves. To do this, we catch them and dispatch them to the + uncaught exception handler via a forked thread. This is similar to our + handling in Netty. + +commit 2b3cb240f22bbc75b9e3be2c7a4f3b19b1b30f88 +Author: Tal Levy +Date: Mon Aug 20 15:44:47 2018 -0700 + + add INDEX_LIFECYCLE_ORIGIN to run as xpack + +commit cd83ddceccbf8cfbcddd9e1a72db7b98101f90e9 +Author: Tim Brooks +Date: Mon Aug 20 16:09:22 2018 -0600 + + Fix assertion in AbstractSimpleTransportTestCase (#32991) + + This is a follow-up to #32956. That commit incorrectly used assertBusy + which led to a possible race in the test. This commit fixes it. + +commit 2feda8aae0ccd39c5c01ee7a232907a0eb499ab8 +Author: Lisa Cawley +Date: Mon Aug 20 14:30:42 2018 -0700 + + [DOC] Splits role mapping APIs into separate pages (#32797) + +commit 3fbaae10af1a8ed26d7d1dea2b51ad8371036223 +Author: Benjamin Trent +Date: Mon Aug 20 16:05:56 2018 -0500 + + HLRC: ML Close Job (#32943) + + * HLRC: Adding ML Close Job API + + HLRC: Adding ML Close Job API + + * reconciling request converters + + * Adding serialization tests and addressing PR comments + + * Changing constructor order + +commit 77d7547be2782f5157b7dbb88aa805f8e9593413 +Author: Nhat Nguyen +Date: Mon Aug 20 16:33:15 2018 -0400 + + Fix compilation after merge from master + +commit 853eb1c51cd042acb0ea49d1bad97c5fa9bfc444 +Merge: ac75968c0b0 9050c7e846b +Author: Jason Tedor +Date: Mon Aug 20 15:49:31 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Generalize remote license checker (#32971) + Trim translog when safe commit advanced (#32967) + Fix an inaccuracy in the dynamic templates documentation. (#32890) + Logging: Use settings when building daemon threads (#32751) + All Translog inner closes should happen after tragedy exception is set (#32674) + HLREST: AwaitsFix ML Test + Pass DiscoveryNode to initiateChannel (#32958) + Add mzn and dz to unsupported locales (#32957) + Use settings from the context in BootstrapChecks (#32908) + Update docs for node specifications (#30468) + HLRC: Forbid all Elasticsearch logging infra (#32784) + Only configure publishing if it's applied externally (#32351) + Fixes libs:dissect when in eclipse + Protect ScriptedMetricIT test cases against failures on 0-doc shards (#32959) (#32968) + [Kerberos] Add documentation for Kerberos realm (#32662) + Watcher: Properly find next valid date in cron expressions (#32734) + Fix some small issues in the getting started docs (#30346) + Set forbidden APIs target compatibility to compiler java version (#32935) + Move connection listener to ConnectionManager (#32956) + +commit 9050c7e846b6a34e7a33e05418b29f5789154943 +Author: Jason Tedor +Date: Mon Aug 20 15:33:29 2018 -0400 + + Generalize remote license checker (#32971) + + Machine learning has baked a remote license checker for use in checking + license compatibility of a remote license. This remote license checker + has general usage for any feature that relies on a remote cluster. For + example, cross-cluster replication will pull changes from a remote + cluster and require that the local and remote clusters have platinum + licenses. This commit generalizes the remote cluster license check for + use in cross-cluster replication. + +commit 40f1bb5e5eab4445f0113131f415963664a1c530 +Author: Nhat Nguyen +Date: Mon Aug 20 15:13:19 2018 -0400 + + Trim translog when safe commit advanced (#32967) + + Since #28140 when the global checkpoint is advanced, we try to move the + safe commit forward, and clean up old index commits if possible. However, + we forget to trim unreferenced translog. + + This change makes sure that we prune both old translog and index commits + when the safe commit advanced. + + Relates #28140 + Closes #32089 + +commit 815c56b6770cb9a4b1a58dd831655cd312df09b6 +Author: Julie Tibshirani +Date: Mon Aug 20 11:00:11 2018 -0700 + + Fix an inaccuracy in the dynamic templates documentation. (#32890) + +commit 462e91d362c9ce8aa807a702081f9bbd76f8d019 +Author: Nik Everett +Date: Mon Aug 20 13:53:15 2018 -0400 + + Logging: Use settings when building daemon threads (#32751) + + Subclasses of `EsIntegTestCase` run multiple Elasticsearch nodes in the + same JVM and when we log we look at the name of the thread to figure out + the node name. This makes sure that all calls to `daemonThreadFactory` + include the node name. + + Closes #32574 + + I'd like to follow this up with more drastic changes that make it + impossible to do this incorrectly but that change is much larger than + this and I'd like to get these log lines fixed up sooner rather than + later. + +commit 0749b18181c8dbfc3f85a4a6d64dd09beeb411ac +Author: Andrey Ershov +Date: Mon Aug 20 19:22:10 2018 +0200 + + All Translog inner closes should happen after tragedy exception is set (#32674) + + All Translog inner closes should happen after tragedy exception is set (#32674) + + We faced with the nasty race condition. See #32526 + InternalEngine.failOnTragic method has thrown AssertionError. + If you carefully look at if branches in this method, you will spot that its only possible, if either Lucene IndexWriterhas closed from inside or Translog, has closed from inside, but tragedy exception is not set. + For now, let us concentrate on the Translog class. + We found out that there are two methods in Translog - namely rollGeneration and trimOperations that are closing Translog in case of Exception without tragedy exception being set. + This commit fixes these 2 methods. To fix it, we pull tragedyException from TranslogWriter up-to Translog class, because in these 2 methods IndexWriter could be innocent, but still Translog needs to be closed. Also, tragedyException is wrapped with TragicExceptionHolder to reuse CAS/addSuppresed functionality in Translog and TranslogWriter. + Also to protect us in the future and make sure close method is never called from inside Translog special assertion examining stack trace is added. Since we're still targeting Java 8 for runtime - no StackWalker API is used in the implementation. + In the stack-trace checking method, we're considering inner caller not only Translog methods but Translog child classes methods as well. It does mean that Translog is meant for extending it, but it's needed to be able to test this method. + + Closes #32526 + +commit 34295fad8713c5fcf6393085bb3dd368146d3ca2 +Author: Nik Everett +Date: Mon Aug 20 13:05:55 2018 -0400 + + HLREST: AwaitsFix ML Test + + It leaks state into other tests causing them to fail sometimes. + + Relates to #32993 + +commit cd6326b391afe6e06044bf061f4ed8aae6b68a46 +Author: David Turner +Date: Mon Aug 20 17:48:05 2018 +0100 + + Introduce PreVoteCollector (#32847) + + An election requires a node to select a term that is higher than all + previously-seen terms. If nodes are too enthusiastic about starting elections + then they can effectively excludes itself from the cluster until the leader can + bump to a still-higher term, and if this process repeats then a single faulty + node can prevent the cluster from making useful progress. + + The solution is to start the election with a pre-voting round to ensure that + there is at least a quorum of nodes who believe there to be no leader. + + This also fixes up some merge issues. + +commit 5ce082cd0aa02995df7cee95d3d6594bb3e33884 +Author: Tal Levy +Date: Mon Aug 20 08:32:22 2018 -0700 + + copy LifecyclePolicy to protocol.xpack (#32915) + + This is the final PR for copying over the necessary components for + clients to parse/render LifecyclePolicy. Changes include: + + - move of named-x-content server objects away from client + - move validation into the client copy of LifecyclePolicy + - move LifecycleAction into an interface with `getName` + +commit faa42de66d67f6ddcdd0e89c8bfb6cde96c11471 +Author: Tim Brooks +Date: Mon Aug 20 08:54:55 2018 -0600 + + Pass DiscoveryNode to initiateChannel (#32958) + + This is related to #32517. This commit passes the DiscoveryNode to the + initiateChannel method for different Transport implementation. This + will allow additional attributes (besides just the socket address) to be + used when opening channels. + +commit eef0e35913007d4068d06d5274b8e1f24251b7d9 +Author: Ioannis Kakavas +Date: Mon Aug 20 17:12:02 2018 +0300 + + Add mzn and dz to unsupported locales (#32957) + + Add mzn and dz to the list of unsupported locales + for Kerberos tests. + +commit 6905ca9d6cbc9b1227b07f252f7c3598fb5a6f57 +Author: Ioannis Kakavas +Date: Mon Aug 20 17:01:10 2018 +0300 + + Use settings from the context in BootstrapChecks (#32908) + + Use settings from the context in BootstrapChecks + instead of passing them in the constructor + +commit a883e7dffcb96750cc961803d15c2e7de27f915d +Author: Yu +Date: Mon Aug 20 15:21:31 2018 +0200 + + Update docs for node specifications (#30468) + + Expands and clarifies exactly what is and isn't allowed when specifying a + subset of the nodes as targets of a cluster API, and adds missing links to this + from the hot threads and cluster stats API docs. + + Co-authored-by: David Turner + Co-authored-by: Yu + +commit f853f6f03c7eb0590def1be73458d33ccbe50087 +Author: Nik Everett +Date: Mon Aug 20 08:55:24 2018 -0400 + + HLRC: Forbid all Elasticsearch logging infra (#32784) + + All of the Elasticsearch logging infrastructure relies on log4j but we + don't want the high level rest client to rely on log4j2. All of its + logging goes through commons-logging because our dependencies drag in + commons logging already. Anyway, this bans direct use of Elasticsearch's + logging infrastructure in the high level REST client. It is still + possible to use it indirectly though and there isn't anything we can + really do about that until we split the high level rest client from + Elasticsearch's server jar. + +commit e3700a9b8dec77691bc69b1e90c69198912c9f63 +Author: Alpar Torok +Date: Mon Aug 20 13:46:21 2018 +0300 + + Only configure publishing if it's applied externally (#32351) + + Only configure publishing if it's applied externally, reconfigure for hasClientJar + +commit 3736097e199717cf86481e021bdc60573c88e318 +Merge: 77016add191 b6e95cde3ac +Author: Colin Goodheart-Smithe +Date: Mon Aug 20 10:06:02 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit b6e95cde3ace2355ddc03368c2821eccffea76cc +Author: Colin Goodheart-Smithe +Date: Mon Aug 20 10:05:25 2018 +0100 + + Fixes libs:dissect when in eclipse + +commit f6891cd222b29867882e1b768ba68f6aeffc9268 +Author: David Turner +Date: Mon Aug 20 08:58:03 2018 +0100 + + Fixup after merge + +commit 676091aafbdd7482118a71768a0dffa2889e2218 +Author: Jonathan Little +Date: Mon Aug 20 00:55:43 2018 -0700 + + Protect ScriptedMetricIT test cases against failures on 0-doc shards (#32959) (#32968) + + Randomized test conditions that cause some shards to have no docs on them + failed due to test asserts that relied on a lazy initialization side effect + from the map script. After this fix: + + - Test cases with the relevant init script are protected + - Test cases with the relevant combine or reduce scripts were already + protected, because the combine and reduce scripts safely handle this case. + +commit f317562c824c16fa11c7de073d3449081fa95147 +Merge: a3bb85eeaf8 dce72c79850 +Author: David Turner +Date: Mon Aug 20 08:33:55 2018 +0100 + + Merge branch 'master' into zen2 + +commit e143cce86594eae12da38bfb62de8005ac5dfea2 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Aug 20 17:23:14 2018 +1000 + + [Kerberos] Add documentation for Kerberos realm (#32662) + + This commit adds documentation for configuring Kerberos realm. + Configuring Kerberos realm documentation highlights important + terminology and requirements before creating Kerberos realm. + Most of the documentation is centered around configuration from + Elasticsearch rather than go deep into Kerberos implementation. + Kerberos realm settings are mentioned in the security settings + for Kerberos realm. + +commit 3fa36807f87ad90af593e86f9ed843ced3260973 +Author: Alexander Reelsen +Date: Mon Aug 20 09:18:51 2018 +0200 + + Watcher: Properly find next valid date in cron expressions (#32734) + + When a list/an array of cron expressions is provided, and one of those addresses + is already expired, the expired one will be considered as an option + instead of the valid next one. + + This commit also reduces the visibility of the CronnableSchedule and + refactors a comparator to look like java 8. + +commit dce72c798508ce48e91a50f684113868f8fc9a25 +Author: Tim Ryan +Date: Mon Aug 20 02:54:03 2018 -0400 + + Fix some small issues in the getting started docs (#30346) + + * Modified a reference to real time to match the previous line reference of + realtime. + * Modified eg to e.g. as it's an abbreviation for the latin exempli gratia + * Added missing pronoun to `_executing_filters` section. + +commit 4b34b3f4aade488de01c7023670b6360ac7e28da +Author: Alpar Torok +Date: Mon Aug 20 09:27:02 2018 +0300 + + Set forbidden APIs target compatibility to compiler java version (#32935) + + Set forbidden apis target compatibility to compiler version + + Fix outstanding deprecation + +commit de92d2ef1fb03dda6f07082f8e87dde1ce6a663b +Author: Tim Brooks +Date: Sat Aug 18 10:09:24 2018 -0600 + + Move connection listener to ConnectionManager (#32956) + + This is a followup to #31886. After that commit the + TransportConnectionListener had to be propogated to both the + Transport and the ConnectionManager. This commit moves that listener + to completely live in the ConnectionManager. The request and response + related methods are moved to a TransportMessageListener. That listener + continues to live in the Transport class. + +commit ac75968c0b0e40d0e7dc478a1f52821f53866585 +Merge: 4475f88c95f f82bb64feba +Author: Jason Tedor +Date: Sat Aug 18 09:48:55 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: (46 commits) + NETWORKING: Make RemoteClusterConn. Lazy Resolve DNS (#32764) + [DOCS] Splits the users API documentation into multiple pages (#32825) + [DOCS] Splits the token APIs into separate pages (#32865) + [DOCS] Creates redirects for role management APIs page + Bypassing failing test PainlessDomainSplitIT#testHRDSplit (#32966) + TEST: Mute testRetentionPolicyChangeDuringRecovery + [DOCS] Fixes more broken links to role management APIs + [Docs] Tweaks and fixes to rollup docs + [DOCS] Fixes links to role management APIs + [ML][TEST] Fix BasicRenormalizationIT after adding multibucket feature + [DOCS] Splits the roles API documentation into multiple pages (#32794) + [TEST] Run pre 6.4 nodes in non-FIPS JVMs (#32901) + Make Geo Context Mapping Parsing More Strict (#32821) + [ML] fix updating opened jobs scheduled events (#31651) (#32881) + Scripted metric aggregations: add deprecation warning and system property to control legacy params (#31597) + Tests: Fix timezone conversion in DateTimeUnitTests + Enable FIPS140LicenseBootstrapCheck (#32903) + Fix InternalAutoDateHistogram reproducible failure (#32723) + Remove assertion in testDocStats on deletedDocs counter (#32914) + HLRC: Move ML request converters into their own class (#32906) + ... + +commit f82bb64feba3d3420dd790d02cb7dd5ee612df4d +Author: Armin Braun +Date: Sat Aug 18 08:46:44 2018 +0200 + + NETWORKING: Make RemoteClusterConn. Lazy Resolve DNS (#32764) + + * Lazy resolve DNS (i.e. `String` to `DiscoveryNode`) to not run into indefinitely caching lookup issues (provided the JVM dns cache is configured correctly as explained in https://www.elastic.co/guide/en/elasticsearch/reference/6.3/networkaddress-cache-ttl.html) + * Changed `InetAddress` type to `String` for that higher up the stack + * Passed down `Supplier` instead of outright `DiscoveryNode` from `RemoteClusterAware#buildRemoteClustersSeeds` on to lazy resolve DNS when the `DiscoveryNode` is actually used (could've also passed down the value of `clusterName = REMOTE_CLUSTERS_SEEDS.getNamespace(concreteSetting)` together with the `List` of hosts, but this route seemed to introduce less duplication and resulted in a significantly smaller changeset). + * Closes #28858 + +commit 532d552ffd61751f18778596e32c3e0d9ca1bd80 +Author: Lisa Cawley +Date: Fri Aug 17 23:17:33 2018 -0700 + + [DOCS] Splits the users API documentation into multiple pages (#32825) + +commit fb1c3990d72aa1b2272db9e37c06dfc7a422da77 +Author: Lisa Cawley +Date: Fri Aug 17 22:22:09 2018 -0700 + + [DOCS] Splits the token APIs into separate pages (#32865) + +commit 1efee66d164d1d6996bae95ab7c9f4cca9c98d5e +Author: lcawl +Date: Fri Aug 17 21:39:21 2018 -0700 + + [DOCS] Creates redirects for role management APIs page + +commit 77016add19158fd6ea80099915c61a5e738266b1 +Author: Lee Hinman +Date: Fri Aug 17 22:45:15 2018 -0600 + + Store phase steps for index in PolicyStepsRegistry (#32926) + + * Store phase steps for index in PolicyStepsRegistry + + This changes the way that steps are retrieved from `PolicyStepsRegistry` to + store the steps on a per-index basis (in memory for now, though that will change + in subsequent PRs). These steps are rebuilt as the index changes phases. + + This also fixes a bug where an action with the same phase and name was not being + considered changed (and thus updated) in the compiled steps list. These are now + correctly considered as "upsert" diffs. + + Relates to #29823 + +commit a26e10859092dc6da87df700328fa3c91f238794 +Merge: 711ac402b66 647705e00a1 +Author: Tal Levy +Date: Fri Aug 17 13:57:28 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 647705e00a118bbafb3258d451beeee9a26c90bc +Author: Benjamin Trent +Date: Fri Aug 17 15:30:31 2018 -0500 + + Bypassing failing test PainlessDomainSplitIT#testHRDSplit (#32966) + +commit 86ffce4bbc93eadb8b3b505ef48b8bf0897cb69b +Author: Nhat Nguyen +Date: Fri Aug 17 13:55:42 2018 -0400 + + TEST: Mute testRetentionPolicyChangeDuringRecovery + + Tracked at #32089 + +commit 967b1785fab89fc735791e64ec23cbd993a5e989 +Author: lcawl +Date: Fri Aug 17 10:41:06 2018 -0700 + + [DOCS] Fixes more broken links to role management APIs + +commit 899e94a29bc8a4e7a9497caf12e5ec153d8f8870 +Author: Zachary Tong +Date: Fri Aug 17 13:33:12 2018 -0400 + + [Docs] Tweaks and fixes to rollup docs + + - Missing links to new IndexCaps API + - Incorrect security permissions on IndexCaps API + - GetJobs API must supply a job (or `_all`), omitting throws error + - Link to search/agg limitations from RollupSearch API + - Tweak URLs in quick reference + - Formatting of overview page + +commit a608205510519c3ec7a8554c1f8f16744a6ccfce +Author: lcawl +Date: Fri Aug 17 10:20:28 2018 -0700 + + [DOCS] Fixes links to role management APIs + +commit 46c35db1dfb7fa59f47a420545ab76f342ad701d +Author: Dimitris Athanasiou +Date: Fri Aug 17 17:52:29 2018 +0100 + + [ML][TEST] Fix BasicRenormalizationIT after adding multibucket feature + + As the multibucket feature was merged in, this test hit a side effect + which means buckets trailing an anomaly could become anomalous. + This commit fixes the problem by filtering low score records when + we request them. + +commit c5de9ec79d4e60714761765d4581de863ed97f64 +Author: Lisa Cawley +Date: Fri Aug 17 09:18:08 2018 -0700 + + [DOCS] Splits the roles API documentation into multiple pages (#32794) + +commit e3aa68b0a9f72db7da5ec1fbba48da87edad6c7d +Author: Ioannis Kakavas +Date: Fri Aug 17 18:23:13 2018 +0300 + + [TEST] Run pre 6.4 nodes in non-FIPS JVMs (#32901) + + Elasticsearch versions earlier than 6.4.0 cannot properly run in a + FIPS 140 JVM. This commit ensures that we use a non-FIPS JVM for + nodes that we spin up in BWC tests even when we're testing FIPS. + +commit da6b61e8ef912fcb89c03e69f322ee517d879f06 +Author: Igor Motov +Date: Fri Aug 17 08:13:16 2018 -0700 + + Make Geo Context Mapping Parsing More Strict (#32821) + + Currently, if geo context is represented by something other than + geo_point or an object with lat and lon fields, the parsing of it + as a geo context can result in ignoring the context altogether, + returning confusing errors such as number_format_exception or trying + to parse the number specifying as long-encoded hash code. It would also + fail if the geo_point was stored. + + This commit makes the mapping parsing more strict and will fail during + mapping update or index creation if the geo context doesn't point to + a geo_point field. + + Supersedes #32412 + + Closes #32202 + +commit 9cec4aa14b7ea2a4283348df9d5a45cf1a5ab137 +Author: Benjamin Trent +Date: Fri Aug 17 07:21:17 2018 -0500 + + [ML] fix updating opened jobs scheduled events (#31651) (#32881) + + * ML: fix updating opened jobs scheduled events (#31651) + + * Adding UpdateParamsTests license header + + * Adding integration test and addressing PR comments + + * addressing test and job names + +commit a08127c072915b8598bbbf4ed7302aa737cddd6c +Author: Jonathan Little +Date: Fri Aug 17 05:11:18 2018 -0700 + + Scripted metric aggregations: add deprecation warning and system property to control legacy params (#31597) + + * Scripted metric aggregations: add deprecation warning and system property to control legacy params + + Scripted metric aggregation params._agg/_aggs are replaced by state/states context variables. By default the old params are still present, and a deprecation warning is emitted when Scripted Metric Aggregations are used. A new system property can be used to disable the legacy params. This functionality will be removed in a future revision. + + * Fix minor style issue and docs test failure + + * Disable deprecated params._agg/_aggs in tests and revise tests to use state/states instead + + * Add integration test covering deprecated scripted metrics aggs params._agg/_aggs access + + * Disable deprecated params._agg/_aggs in docs integration tests and revise stored scripts to use state/states instead + + * Revert unnecessary migrations doc change + + A relevant note should be added in the changes destined for 7.0; this PR is going to be backported to 6.x. + + * Replace deprecated _agg param bwc integration test with a couple of unit tests + + * Fix compatibility test after merge + + * Rename backwards compatibility system property per code review feedback + + * Tweak deprecation warning text per review feedback + +commit 0d92f377fd3ecc9c6336f4770fc92951368a2aec +Author: Alexander Reelsen +Date: Fri Aug 17 14:09:01 2018 +0200 + + Tests: Fix timezone conversion in DateTimeUnitTests + + This fix prevernts trying to parse unknown timezone ids by converting + the joda time zone via java.util.TimeZone to a java time based ZoneId. + + Closes #32927 + +commit 75014a22d726cc95cdd00d9cb5a59924049d56a6 +Author: Ioannis Kakavas +Date: Fri Aug 17 14:06:24 2018 +0300 + + Enable FIPS140LicenseBootstrapCheck (#32903) + + This commit ensures that xpack.security.fips_mode.enabled: true + cannot be set in a node that doesn't have the appropriate license. + +commit ca54aacbb5a8603ec14916c4d65da69086dee8ae +Author: Paul Sanwald +Date: Fri Aug 17 07:03:25 2018 -0400 + + Fix InternalAutoDateHistogram reproducible failure (#32723) + + Update test logic to correctly bucket intervals. + +commit 2fa028cfa12067d81ac4bef875b5f8fa84bac5ac +Author: Andrey Ershov +Date: Fri Aug 17 12:36:45 2018 +0200 + + Remove assertion in testDocStats on deletedDocs counter (#32914) + + testDocStats test is flaky and sometimes it's failing on jenkins and + failure is not reproducible locally. The reason for this failure is in + timing. If the number of deleted documents is greater than 33% of inserted + documents, Lucene will schedule segments to merge if TieredMergePolicy is + used (it's not the case for LogMergePolicy, but ES is only using + TieredMergePolicy). If this merge is performed before stats are + retrieved - we will get 0 for "deleted" counter. + So basically this counter could be either 0 or numOfDeletedDocs at this point, + but this is the too loose assertion and we decided to remove it at all. + Closes #32766 + +commit 76aba8ad7bf4a5c8bce78fbefdfbd9d987294e92 +Author: Dimitris Athanasiou +Date: Fri Aug 17 10:57:00 2018 +0100 + + HLRC: Move ML request converters into their own class (#32906) + +commit ae38cfbaec24ed4ab5a28f091e2c9173ab06ea58 +Author: markwalkom +Date: Fri Aug 17 19:03:09 2018 +1000 + + [DOCS] Update getting-started.asciidoc (#29518) + + Highlighted that you can change shard counts using `_shrink` and `_split`. + +commit dd5a5aab88edc88f0f7949ecda67432efdcfc616 +Author: JB Nizet +Date: Fri Aug 17 10:59:26 2018 +0200 + + Fix allowed value for HighlighterBuilder encoder in javadocs (#32780) + + Relates to #32745 + +commit efdad7d5fc32b0887a6fa2f34b259dfb566cf77b +Author: JeffSaxeVA <42005772+JeffSaxeVA@users.noreply.github.com> +Date: Fri Aug 17 04:56:06 2018 -0400 + + [DOCS] Add "remove a tag" script logic as an example (#32556) + + It took me quite a while of online searching and experimenting to realize the function-call asymmetry in the Add versus Remove from a list, like the "tags" list! I realize we cannot give examples for every single thing the user wants to do in Painless, but this is such a common use case (removing a tag from a single doc, or from a set of docs with Update-By-Query) that I believe it ought to be demonstrated immediately after the "add a tag" example. We have an example of removing an entire document field, but not removing one element of a list (a multi-valued field). + + Also, a minor grammar fix: I have added an apostrophe to the word "its" in the accompanying text of the example just above. + +commit 148a76f0c7498ca55c9b56c605578fdb04bded30 +Author: Alpar Torok +Date: Fri Aug 17 11:14:58 2018 +0300 + + Fix failing BuildExamplePluginsIT test + +commit d16562eab5c514f2caa9092b73c53e3a82f7d66d +Author: Alpar Torok +Date: Fri Aug 17 09:41:39 2018 +0300 + + RFC: Test that example plugins build stand-alone (#32235) + + Add tests for build-tools to make sure example plugins build stand-alone using it. + + This will catch issues such as referencing files from the buildSrc directly, breaking external uses of build-tools. + +commit 1136a9583792ae20ffc2d6c58324ce4e04962976 +Author: Jay Modi +Date: Thu Aug 16 21:16:06 2018 -0600 + + Security: remove put privilege API (#32879) + + This commit removes the put privilege API in favor of having a single API to + create and update privileges. If we see the need to have an API like this in + the future we can always add it back. + +commit cbf160a4e6c6c13f63f2d0d2dd1adf006fc70d97 +Author: Julie Tibshirani +Date: Thu Aug 16 17:36:58 2018 -0700 + + For filters aggs, make sure that rewrites preserve other_bucket. (#32921) + +commit 711ac402b6676f13fc8ee6a8d10650642ff1874e +Merge: 62819826c70 0876630b306 +Author: Tal Levy +Date: Thu Aug 16 15:08:22 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 0876630b3069c2d37005c21e06f33980a4541422 +Author: Jason Tedor +Date: Thu Aug 16 16:25:34 2018 -0400 + + Guard against null in email admin watches (#32923) + + The Kibana settings docs that these watches rely on can sometimes + contain no xpack settings. When this is the case, we will end up with a + null pointer exception in the script. We need to guard against in these + scripts so this commit does that. + +commit a3bb85eeaf856aba23277770bd5aa66a595450ee +Author: Yannick Welsch +Date: Thu Aug 16 22:19:17 2018 +0200 + + Zen2: Extract JoinTaskExecutor (#32911) + + Moves JoinTaskExecutor out of ZenDiscovery so that it can be reused for Zen2. Also ensures that tasks to JoinTaskExecutor have a proper identity, so that multiple tasks for the same node can coexist. + +commit 62559d2b3c26c5617049c1c328d0e5522680e96f +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Thu Aug 16 18:23:26 2018 +0100 + + Re enable ml bwc tests (#32916) + + [ML] Re-enabling BWC tests + + Re-enable BWC tests for ML now that #32816 has been backported to 6.x + +commit d604b3e3a1aca100c4560b4d05f846148b76fef4 +Author: Ed Savage +Date: Thu Aug 16 17:18:51 2018 +0100 + + Temporarily disabled ML BWC tests for backporting + + https://github.com/elastic/elasticsearch/pull/32816 + +commit 62819826c704ff69a64e26506788d71cf7aaeb1e +Author: Tal Levy +Date: Thu Aug 16 08:48:26 2018 -0700 + + copy more actions to protocol.xpack (#32892) + + this PR creates client-side instances of the following actions: + + - DeleteAction + - ForceMergeAction + - ReadOnlyAction + - RolloverAction + - ForceMergeAction + + AllocateAction was done separately + +commit c9de707f58fcc9a75e47c3d84f5278ea4cf2c0c6 +Merge: 238eb93893f b5a85362e30 +Author: Tal Levy +Date: Thu Aug 16 08:41:57 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 238eb93893f4bcb43307ecef37b6689fc25d3c6d +Author: Tal Levy +Date: Thu Aug 16 08:41:07 2018 -0700 + + migrate allocate action pojo/xcontent to xpack.protocol (#32853) + +commit b5a85362e30632f256dd4f2b8842c55ac64482f6 +Author: Jason Tedor +Date: Thu Aug 16 11:32:35 2018 -0400 + + Fix docs for fixed filename for heap dump path (#32882) + + The docs here incorrectly state that it is okay for a heap dump file to + exist when heap dump path is configured to a fixed filename. This is + incorrect, the JVM will fail to write the heap dump if a heap dump file + already exists at the specified location (see the DumpWriter constructor + DumpWriter::DumpWriter(const char* path) in the JVM source). + +commit d9fd74bcdcabd5020e40ef7e5dd67c6de09ae0f0 +Author: Jack Conradson +Date: Thu Aug 16 08:03:21 2018 -0700 + + Painless: Special Case def (#32871) + + This removes def from the classes map in PainlessLookup and instead always special + cases it. This prevents potential calls against the def type that shouldn't be made and + forces all cases of def throughout Painless code to be special cased. + +commit eaaf37a1f99309c8a7c176c9b2cc7f56ec5c69d1 +Author: Adrien Grand +Date: Thu Aug 16 15:25:51 2018 +0200 + + AwaitFix FullClusterRestartIT#testRollupIDSchemeAfterRestart. + +commit 3dd1677cdc8fa6b4c36a79f7b206430661469ce8 +Author: Jim Ferenczi +Date: Thu Aug 16 15:33:17 2018 +0200 + + [Test] Fix DuelScrollIT#testDuelIndexOrderQueryThenFetch + + This commit disables the automatic `refresh_interval` in order to ensure + that index readers cannot differ between the normal and scroll search. + This issue is related to the 7.5 Lucene upgrade which contains a change that + makes single segment merge more likely to occur (max deletes percentage). + + Closes #32682 + +commit aedc2c1c498688970afcebe08726480e611f117b +Author: Benjamin Trent +Date: Thu Aug 16 07:18:43 2018 -0500 + + HLRC: adding machine learning delete job (#32820) + + * HLRC: adding machine learning delete job + + * Fixing whitespace + + * Moving docs and tests around + + * Unifying ml asciidoc file naming convention + +commit b87f3062b77cab7888e9037b4996f2c26db12816 +Author: Hazem Khaled +Date: Thu Aug 16 14:54:04 2018 +0300 + + [DOCS] Update WordPress plugins links (#32194) + +commit f8c7414ee861b441f81716e13c3190f4d2d97c54 +Author: Jason Tedor +Date: Thu Aug 16 07:24:05 2018 -0400 + + Remove passphrase support from reload settings API (#32889) + + We do not support passphrases on the secure settings storage (the + keystore). Yet, we added support for this in the API layer. This commit + removes this support so that we are not limited in our future options, + or have to make a breaking change. + +commit e35be019014aafba9c34484e5711a04ce482ffd1 +Author: Adrien Grand +Date: Thu Aug 16 12:27:21 2018 +0200 + + AwaitFix AckIT. + + Relates #32767 + +commit d80457ee2a0a5ce8973b48a06b5d60ee2c5e64c1 +Author: Colin Goodheart-Smithe +Date: Thu Aug 16 11:07:20 2018 +0100 + + Mutes test in DuelScrollIT + + Due to https://github.com/elastic/elasticsearch/issues/32682 + +commit 039babddf532c918ae3f0ac5360e85e7ba5b3052 +Author: Adrien Grand +Date: Thu Aug 16 11:53:01 2018 +0200 + + CharArraysTests: Fix test bug. + +commit 7f6802cb51eda172c64fb223bf5728719f586b11 +Author: David Kyle +Date: Thu Aug 16 10:48:56 2018 +0100 + + [ML] Choose seconds to fix intermittent DatafeeedConfigTest failure + +commit 996ed73d735b1a2943cb5c7af6ba315a225b64c0 +Author: Jim Ferenczi +Date: Thu Aug 16 11:44:20 2018 +0200 + + Test: Fix unpredictive merges in DocumentSubsetReaderTests + + The merge policy that was used could lead to unpredictive merges due to the + randomization of `setDeletesPctAllowed`. + + Closes #32457 + +commit e6bfba1d799cbc41fce5ea88e2252f78fd9cac65 +Author: datosh +Date: Thu Aug 16 11:34:41 2018 +0200 + + [DOCS] Clarify sentence in network-host.asciidoc (#32429) + +commit 70d80a3d093e327a05eeb7b8c49fee7c94952a8f +Author: markharwood +Date: Thu Aug 16 10:21:37 2018 +0100 + + Docs enhancement: added reference to cluster-level setting `search.default_allow_partial_results` (#32810) + + Closes #32809 + +commit 9406aa5af9d26c4fd5007d7133af5a65f3f6976e +Author: Tal Levy +Date: Wed Aug 15 21:55:18 2018 -0700 + + remove extraneous CreateIndexRequest context in ShrinkAction (#32893) + +commit 33522d4fb4d575f969ae5b1e6b34930560d0bc8a +Author: Tal Levy +Date: Wed Aug 15 19:30:21 2018 -0700 + + update cluster-state task execution to halt on new phase (#32886) + + As we migrate to a per-phase execution model, we need to prepare our cluster-state-step execution model to be aligned. It is the case that the final iteration into the next "currentStep" from the next phase would not be available in the registry yet. This change exits the execution loop early as to not jump into executing the next phase's steps before the registry is properly updated + +commit 93ada98eb54d99b9d98ed51778bd6422d18fd392 +Author: debadair +Date: Wed Aug 15 15:09:41 2018 -0700 + + [DOCS] Fixing cross doc link to Stack Overview security topic. + +commit 4baa721459666356f2e4113f8154818fa04f21c7 +Author: Tal Levy +Date: Wed Aug 15 14:47:22 2018 -0700 + + remove `type` config from LifecyclePolicy JSON (#32660) + + Since there is only one production policy, Timeseries, there + is no reason to expose the `type` argument to the user. + +commit d1ab5dd65015da58e7315ccfea5628c59361bf53 +Author: Tal Levy +Date: Wed Aug 15 14:46:31 2018 -0700 + + fix merge with master + +commit 1a45b27d8b692853036ea3acafd50e873d8d0e85 +Author: Jay Modi +Date: Wed Aug 15 15:26:00 2018 -0600 + + Move CharArrays to core lib (#32851) + + This change cleans up some methods in the CharArrays class from x-pack, which + includes the unification of char[] to utf8 and utf8 to char[] conversions that + intentionally do not use strings. There was previously an implementation in + x-pack and in the reloading of secure settings. The method from the reloading + of secure settings was adopted as it handled more scenarios related to the + backing byte and char buffers that were used to perform the conversions. The + cleaned up class is moved into libs/core to allow it to be used by requests + that will be migrated to the high level rest client. + + Relates #32332 + +commit 4475f88c95f0ebfdfe305541542c27e09c9c66e4 +Merge: aa147cca441 364ccc36d60 +Author: Jason Tedor +Date: Wed Aug 15 16:07:28 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Fix global checkpoint listeners test + HLRC: adding machine learning open job (#32860) + [ML] Add log structure finder functionality (#32788) + INGEST: Add Configuration Except. Data to Metdata (#32322) + +commit ec9375660036b3c0a59cb3db60cd0626e9fc5979 +Merge: 92ecd1d2713 364ccc36d60 +Author: Tal Levy +Date: Wed Aug 15 12:56:01 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 364ccc36d608c3c92176ab2aa65b9d09ca0b3f54 +Author: Jason Tedor +Date: Wed Aug 15 15:53:15 2018 -0400 + + Fix global checkpoint listeners test + + This commit fixes a global checkpoint listeners test wherein we were + expecting an executor to have been used even if there were no + listeners. This is silliness, so this commit adjusts the assertion to + verify that the executor never fires if there are no listeners, and + fires exactly once if there is one or more listeners. + +commit 6d9e7c5cec3bd3a4a73c1168d60ce34a99f54468 +Author: David Turner +Date: Wed Aug 15 20:48:16 2018 +0100 + + Introduce ElectionScheduler (#32846) + + The ElectionScheduler runs while there is no known elected master and is + responsible for scheduling elections randomly, backing off on failure, to + balance the desire to elect a master quickly with the desire to avoid more than + one node starting an election at once. + +commit f90f06ede1ebe09005f22d83e319c9d0eeb33e6b +Author: Benjamin Trent +Date: Wed Aug 15 13:19:41 2018 -0500 + + HLRC: adding machine learning open job (#32860) + + * HLRC: adding machine learning open job + + HLRC: adding ML open job API call + + * Addressing @droberts195 comments and fixing minor style issue + +commit 5ba04e23fce73d6fec7c974a9a018e23dc3000f0 +Author: David Roberts +Date: Wed Aug 15 18:04:21 2018 +0100 + + [ML] Add log structure finder functionality (#32788) + + This change adds a library to ML that can be used to deduce a log + file's structure given only a sample of the log file. + + Eventually this will be used to add an endpoint to ML to make the + functionality available to end users, but this will follow in a + separate change. + + The functionality is split into a library so that it can also be + used by a command line tool without requiring the command line + tool to include all server code. + +commit 986c55b83075184541bc70bfc5546e60586bbb8e +Author: Armin Braun +Date: Wed Aug 15 19:02:19 2018 +0200 + + INGEST: Add Configuration Except. Data to Metdata (#32322) + + * closes #27728 + +commit aa147cca441f6967b911d27142562d2cb31716b0 +Merge: 6556186d9a8 bb3eacf3843 +Author: Jason Tedor +Date: Wed Aug 15 12:30:35 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: + Revert "cluster formation DSL - Gradle integration - part 2 (#32028)" (#32876) + cluster formation DSL - Gradle integration - part 2 (#32028) + Introduce global checkpoint listeners (#32696) + Move connection profile into connection manager (#32858) + [ML] Temporarily disabling rolling-upgrade tests + Use generic AcknowledgedResponse instead of extended classes (#32859) + [ML] Removing old per-partition normalization code (#32816) + Use JDK 10 for 6.4 BWC builds (#32866) + Removed flaky test. Looks like randomisation makes these assertions unreliable. + [test] mute IndexShardTests.testDocStats + Introduce the dissect library (#32297) + Security: remove password hash bootstrap check (#32440) + Move validation to server for put user requests (#32471) + [ML] Add high level REST client docs for ML put job endpoint (#32843) + Test: Fix forbidden uses in test framework (#32824) + Painless: Change fqn_only to no_import (#32817) + [test] mute testSearchWithSignificantTermsAgg + Watcher: Remove unused hipchat render method (#32211) + Watcher: Remove extraneous auth classes (#32300) + Watcher: migrate PagerDuty v1 events API to v2 API (#32285) + +commit bb3eacf3843d1884d81abbd71737df161f9717ac +Author: Alpar Torok +Date: Wed Aug 15 19:17:46 2018 +0300 + + Revert "cluster formation DSL - Gradle integration - part 2 (#32028)" (#32876) + + This reverts commit e680a644345ddcc22ffa0fe5476a87d8e22f3788. + +commit e680a644345ddcc22ffa0fe5476a87d8e22f3788 +Author: Alpar Torok +Date: Wed Aug 15 19:16:10 2018 +0300 + + cluster formation DSL - Gradle integration - part 2 (#32028) + + * Implement Version in java + + - This allows to move all all .java files from .groovy. + - Will prevent eclipse from tangling up in this setup + - make it possible to use Version from Java + + * PR review comments + + * Cluster formation plugin with reference counting + + ``` + > Task :plugins:ingest-user-agent:listElasticSearchClusters + Starting cluster: myTestCluster + * myTestCluster: /home/alpar/work/elastic/elasticsearch/plugins/ingest-user-agent/foo + Asked to unClaimAndStop myTestCluster, since cluster still has 1 claim it will not be stopped + + > Task :plugins:ingest-user-agent:testme UP-TO-DATE + Stopping myTestCluster, since no of claims is 0 + ``` + + - Meant to auto manage the clusters lifecycle + - Add integration test for cluster formation + + * Fix rebase + + * Change to `useCluster` method on task + +commit 068d03f56b4faf61c3ae05a77142dfbb59b7ee5a +Author: Jason Tedor +Date: Wed Aug 15 12:04:24 2018 -0400 + + Introduce global checkpoint listeners (#32696) + + This commit introduces the ability for global checkpoint listeners to be + registered at the shard level. These listeners are notified when the + global checkpoint is updated, and also when the shard closes. To + encapsulate these listeners, we introduce a shard-level component that + handles synchronization of notification and modifications to the + collection of listeners. + +commit 2464b686135eebab5b9a3fdcd2784a9cd16fad2e +Author: Tim Brooks +Date: Wed Aug 15 09:08:33 2018 -0600 + + Move connection profile into connection manager (#32858) + + This is related to #31835. It moves the default connection profile into + the ConnectionManager class. The will allow us to have different + connection managers with different profiles. + +commit 51cece190058e924c4f08155ef594b64e40b677d +Author: Ed Savage +Date: Wed Aug 15 15:48:50 2018 +0100 + + [ML] Temporarily disabling rolling-upgrade tests + + BWC tests disabled while backporting https://github.com/elastic/elasticsearch/pull/32816 + +commit 48281ac5bc5b3952f8621c1e9e680026ad049528 +Author: Lee Hinman +Date: Wed Aug 15 08:06:14 2018 -0600 + + Use generic AcknowledgedResponse instead of extended classes (#32859) + + This removes custom Response classes that extend `AcknowledgedResponse` and do nothing, these classes are not needed and we can directly use the non-abstract super-class instead. + + While this appears to be a large PR, no code has actually changed, only class names have been changed and entire classes removed. + +commit 92ecd1d2713c5de442109508b2fcc999317fc5c1 +Merge: b218b1c68d7 8ce1ab3ed97 +Author: Tal Levy +Date: Wed Aug 15 06:11:25 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 8ce1ab3ed97086df921bb1380d7052f260d7b321 +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Wed Aug 15 13:13:32 2018 +0100 + + [ML] Removing old per-partition normalization code (#32816) + + [ML] Removing old per-partition normalization code + + Per-partition normalization is an old, undocumented feature that was + never used by clients. It has been superseded by per-partition maximum + scoring. + + To maintain communication compatibility with nodes prior to 6.5 it is + necessary to maintain/cope with the old wire format + +commit 7ff37ae2325a941adaca54419c209670aa77cd01 +Author: Jason Tedor +Date: Wed Aug 15 08:01:15 2018 -0400 + + Use JDK 10 for 6.4 BWC builds (#32866) + + The 6.4 release will officially be built with JDK 10. We should push JDK + 10 down to the BWC builds for the 6.4 branch then too. + +commit eb812c8b7dfbcfcfda633c53cd9a485930be4305 +Author: markharwood +Date: Wed Aug 15 10:06:39 2018 +0100 + + Removed flaky test. Looks like randomisation makes these assertions unreliable. + + This test is superfluous - it was added to address #32770 but it later turned out there was an existing test that just required a fix to provide the missing test coverage. + + Closes #32855 + +commit a1cff8601239ccc58b81c688678c27c7f3841f7d +Author: Andy Bristol +Date: Tue Aug 14 18:21:59 2018 -0700 + + [test] mute IndexShardTests.testDocStats + + For #32766 + +commit be620920608c2c19a5954bd3cd36cc0ecb3457eb +Author: Jake Landis +Date: Tue Aug 14 17:08:55 2018 -0700 + + Introduce the dissect library (#32297) + + The dissect library will be used for the ingest node as an alternative + to Grok to split a string based on a pattern. Dissect differs from + Grok such that regular expressions are not used to split the string. + Note - Regular expressions are used during construction of the + objects, but not in the hot path. + + A dissect pattern takes the form of: '%{a} %{b},%{c}' which is + composed of 3 keys (a,b,c) and two delimiters (space and comma). + This dissect pattern will match a string of the form: 'foo bar,baz' + and will result a key/value pairing of 'a=foo, b=bar, and c=baz'. + See the comments in DissectParser for a full explanation. + + This commit does not include the ingest node processor that will consume + it. However, the consumption should be a trivial mapping between the + key/value pairing returned by the parser and the key/value pairing + needed for the IngestDocument. + +commit b218b1c68d7ef42286023efcd87439f55e2170b2 +Author: Tal Levy +Date: Tue Aug 14 12:57:43 2018 -0700 + + introduce random timeseries lifecycle policy util method (#32852) + + It is useful to have a random TimeseriesLifecycleType-backed LifecyclePolicy + for testing. This PR exposes a helper method to create one and use it for serialization tests + in LifecyclePolicyTests + +commit ac5ef8c3892207e65ca6ecb1be331c0cbf566c55 +Author: Jay Modi +Date: Tue Aug 14 13:34:22 2018 -0600 + + Security: remove password hash bootstrap check (#32440) + + This change removes the PasswordHashingBootstrapCheck and replaces it + with validation on the setting itself. This ensures we always get a + valid value from the setting when it is used. + +commit 7d8a64d703bd3d85efd528a961bdafe4fad80d54 +Author: Jay Modi +Date: Tue Aug 14 13:18:07 2018 -0600 + + Move validation to server for put user requests (#32471) + + This change moves the validation for values of usernames and passwords + from the request to the transport action. This is done to prevent + the need to move more classes into protocol once we add this API to the + high level rest client. Additionally, this resolves an issue where + validation depends on settings and we always pass empty settings + instead of the actual settings. + + Relates #32332 + +commit c985f500f4db9c3c4fe46a874484c1f70ead77ff +Author: David Roberts +Date: Tue Aug 14 19:52:54 2018 +0100 + + [ML] Add high level REST client docs for ML put job endpoint (#32843) + + Relates #29827 + Relates #32726 + +commit 0158b59a5a6257b69d61a4f1dd1732cd79fbe8d7 +Author: Ryan Ernst +Date: Tue Aug 14 11:35:09 2018 -0700 + + Test: Fix forbidden uses in test framework (#32824) + + This commit fixes existing uses of forbidden apis in the test framework + and re-enables the forbidden apis check. It was previously completely + disabled and had missed a rename of the forbidden apis signatures files. + + closes #32772 + +commit ab7e7508f5ef52055051363f0a032d27452cf64f +Author: Jack Conradson +Date: Tue Aug 14 11:17:15 2018 -0700 + + Painless: Change fqn_only to no_import (#32817) + + This changes the whitelist parameter fqn_only to no_import when specifying that a + whitelisted class must have the fully-qualified-name instead of a shortcut name. This more + closely correlates with Java imports, hence the rename. + +commit 84aa87f638511c893db1a5c290ecf01e9f20fb2c +Author: Andy Bristol +Date: Tue Aug 14 10:58:20 2018 -0700 + + [test] mute testSearchWithSignificantTermsAgg + + For #32855 + +commit 6556186d9a89a3db77680e77e7bfa17fb3eaa741 +Merge: 919888eba7b cd0de16089e +Author: Nhat Nguyen +Date: Tue Aug 14 12:11:31 2018 -0400 + + Merge branch 'master' into ccr + +commit 6fde9e5c2ffd5f88ca1cf778b76868f78699ac57 +Author: Michael Basnight +Date: Tue Aug 14 11:07:58 2018 -0500 + + Watcher: Remove unused hipchat render method (#32211) + + The HipChatMessage#render is no longer used, and instead the + HipChatAccount#render is used in the ExecutableHipChatAction. Only a + test that validated the HttpProxy used this render method still. This + commit cleans it up. + +commit e1206a6272d1ce2269efa97119071584cc35f44c +Author: Michael Basnight +Date: Tue Aug 14 11:06:46 2018 -0500 + + Watcher: Remove extraneous auth classes (#32300) + + The auth.basic package was an example of a single implementation + interface that leaked into many different classes. In order to clean + this up, the HttpAuth interface, factories, and Registries all were + removed and the single implementation, BasicAuth, was substituted in all + cases. This removes some dependenies between Auth and the Templates, + which can now use static methods on BasicAuth. BasicAuth was also moved + into the http package and all of the other classes were removed. + +commit 4c90a61a35db9963c11afda95000875503b4b868 +Author: Michael Basnight +Date: Tue Aug 14 11:06:18 2018 -0500 + + Watcher: migrate PagerDuty v1 events API to v2 API (#32285) + + The PagerDuty v1 API is EOL and will stop accepting new accounts + shortly. This commit swaps out the watcher use of the v1 API with the + new v2 API. It does not change anything about the existing watcher + API. + + Closes #32243 + +commit cd0de16089effa9e2e1722f133c832c4778dad64 +Author: Yannick Welsch +Date: Tue Aug 14 16:36:37 2018 +0200 + + [TEST] Select free port for Minio (#32837) + + Minio does not support dynamic ports. The workaround here is to scan for a free port first. This is + not foolproof, but as we don't expect too many of these builds to run at once on the same machine, + this should do the trick. + + Closes #32701 + Closes #32208 + +commit 27e64e72516a82ebdb5acfca6b9c62c5fd631cbc +Author: Armin Braun +Date: Tue Aug 14 16:01:33 2018 +0200 + + MINOR: Remove `IndexTemplateFilter` (#32841) + + * This isn't used anywhere anymore ever since `00c123b59f8ba11eb260e6b70acf7be80bccc949` and `dc166c5dc6bcf4abb7f25c6f4143f07d8176333d` + +commit a84b3239c33b19eb54de9dce905b13c41fecd6e5 +Merge: e78f537e585 87481a0e345 +Author: Colin Goodheart-Smithe +Date: Tue Aug 14 13:07:26 2018 +0100 + + Merge branch 'master' into index-lifecycle + + client/rest-high-level/src/main/java/org/elasticsearch/client/RequestCon + verters.java + /Users/colings86/dev/work/git/elasticsearch/.git/worktrees/elasticsearch + -ilm/MERGE_HEAD + + client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseCli + ent.java + client/rest-high-level/src/main/java/org/elasticsearch/client/RequestCon + verters.java + client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.j + ava + client/rest-high-level/src/test/java/org/elasticsearch/client/documentat + ion/LicensingDocumentationIT.java + docs/java-rest/high-level/licensing/delete-license.asciidoc + server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionC + ontext.java + server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.j + ava + server/src/main/java/org/elasticsearch/common/Rounding.java + server/src/main/java/org/elasticsearch/common/rounding/Rounding.java + server/src/main/java/org/elasticsearch/search/aggregations/bucket/signif + icant/ParsedSignificantTerms.java + server/src/test/java/org/elasticsearch/action/IndicesRequestIT.java + server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIn + gestTests.java + server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkAct + ionTests.java + server/src/test/java/org/elasticsearch/common/RoundingTests.java + server/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests + .java + server/src/test/java/org/elasticsearch/common/rounding/RoundingDuelTests + .java + server/src/test/java/org/elasticsearch/search/aggregations/bucket/signif + icant/SignificantLongTermsTests.java + server/src/test/java/org/elasticsearch/search/aggregations/bucket/signif + icant/SignificantStringTermsTests.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicense + Action.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicense + Request.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicense + RequestBuilder.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicense + Response.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseServic + e.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClie + nt.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLic + enseAction.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDele + teLicenseAction.java + x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManag + erServiceTests.java + x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTrans + portTests.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/D + eleteLicenseRequest.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/D + eleteLicenseResponse.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/D + eleteLicenseResponseTests.java + +commit 87481a0e345408427ba0d9e1dbc50c4132f93509 +Author: Alexander Reelsen +Date: Tue Aug 14 13:52:55 2018 +0200 + + Core: Add java time version of rounding classes (#32641) + + This commit adds a java time version of the existing rounding classes, which features the same test suite and a small test class to check if serialization works as expected. + +commit e5ab09f7087f0aabc399bf288adb9e610f016473 +Author: markharwood +Date: Tue Aug 14 11:14:47 2018 +0100 + + Aggregations/HL Rest client fix: missing scores (#32774) + + Significance score doubles were being parsed as long. Existing tests did not catch this because SignificantLongTermsTests and SignificantStringTermsTests did not set the score. Fixed these and also added integration test. + + Thanks for the report/fix, Blakko + + Closes #32770 + +commit 00b006f033a3eaceeeffcce204ae364db9325f0d +Author: Yannick Welsch +Date: Tue Aug 14 11:55:54 2018 +0200 + + HLRC: Add Delete License API (#32586) + + Relates to #29827 + +commit 124c1f13582d0e8be6a25ddc23f7367836bce296 +Author: Armin Braun +Date: Tue Aug 14 11:27:08 2018 +0200 + + INGEST: Create Index Before Pipeline Execute (#32786) + + * INGEST: Create Index Before Pipeline Execute + + * Ensures that indices are created before the default pipeline setting is read to correcly handle the case of an index template containing a default pipeline (without the fix the first document does not get the pipeline applied as explained in #32758) + * closes #32758 + +commit a8bfa466b2f2a68c2384bc730a0b8c7c9ce7ea87 +Author: Yannick Welsch +Date: Tue Aug 14 08:20:35 2018 +0200 + + Fix NOOP bulk updates (#32819) + + #31821 introduced an unreleased bug where NOOP updates were incorrectly mutating the bulk + shard request, inserting null item to be replicated, which would result in NullPointerExceptions when + serializing the request to be shipped to the replicas. + + Closes #32808 + +commit e78f537e5857c7562db06d99cadc6fa396274996 +Merge: 5011aa2c2b0 10fddb62ee3 +Author: Tal Levy +Date: Mon Aug 13 16:31:40 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 10fddb62ee33d4154013883512ca91d14ade69d7 +Author: Tim Brooks +Date: Mon Aug 13 16:44:09 2018 -0600 + + Remove client connections from TcpTransport (#31886) + + This is related to #31835. This commit adds a connection manager that + manages client connections to other nodes. This means that the + TcpTransport no longer maintains a map of nodes that it is connected + to. + +commit 8a003e1281446820db5f73f231767dfcf2a48e72 +Author: Nhat Nguyen +Date: Mon Aug 13 16:29:34 2018 -0400 + + Increase logging testRetentionPolicyChangeDuringRecovery + + Relates #32089 + +commit 5011aa2c2b0ce0f873c81d307706ad05014ffb20 +Author: debadair +Date: Mon Aug 13 13:15:15 2018 -0700 + + Set up ILM docs. (#32823) + + * [DOCS] Added ILM docs to the ref. + + * [DOCS] Added structure for ILM docs. + + * [DOCS] Made the intro a bit more general per Tal's comment. + +commit cba7fceb53c4ef1093b8ae10c19111071b72b1fd +Author: Nhat Nguyen +Date: Mon Aug 13 15:05:42 2018 -0400 + + AwaitsFix case-functions.sql-spec + + Relates #32589 + +commit 06758f579cb6f0e04ed3db99c0563f41726ef312 +Author: Ioannis Kakavas +Date: Mon Aug 13 21:27:06 2018 +0300 + + Mute security-cli tests in FIPS JVM (#32812) + + All Unit tests in this module are muted in FIPS 140 JVMs and + as such the CI run fails. This commit disables test task for the + module in a FIPS JVM and reverts adding a dummy test in + 4cbcc1. + +commit d412230cdaa7a463c218af7f2c90041a9bccb416 +Author: Armin Braun +Date: Mon Aug 13 20:08:26 2018 +0200 + + SCRIPTING: Support BucketAggScript return null (#32811) + + * As explained in #32790, `BucketAggregationScript` must support `null` as a return value + * Closes #32790 + +commit 106a4a6a486b3b29f35dd7ef0f87288edffef3ec +Author: Ioannis Kakavas +Date: Mon Aug 13 21:08:10 2018 +0300 + + Unmute WildFly tests in FIPS JVM (#32814) + + WildflyIT test fails in a FIPS JVM due to the amount of output in stderr. The excessive stderr output is due to https://bugs.openjdk.java.net/browse/JDK-8202893 and is not an indication of a failure that should be tracked. + This commit adjusts the limit to something more lenient that would allow the test to succeed. + Reverts #32543 + +commit 4da0b52a19a91b4a8a195a73a46de7435b4fb3bb +Author: Zachary Tong +Date: Mon Aug 13 12:53:22 2018 -0400 + + [TEST] Force a stop to save rollup state before continuing (#32787) + + We only upgrade the ID when the state is saved in one of four scenarios: + + - when we reach a checkpoint (every 50 pages) + - when we run out of data + - when explicitly stopped + - on failure + + The test was relying on the pre-upgrade to finish, save state and then + the post-upgrade to start, hit the end of data and upgrade ID. THEN + get the new doc and apply the new ID. + + But I think this is vulnerable to timing issues. If the pre-upgrade + portion shutdown before it saved the state, when restarting we would run + through all the data from the beginning with the old ID, meaning both + docs would still have the old scheme. + + This change makes the pre-upgrade wait for the job to go back to STARTED + so that we know it persisted the end point. Post-upgrade, it stops and + restarts the job to ensure the state was persisted and the ID upgraded. + + That _should_ rule out the above timing issue. + + Closes #32773 + +commit a771478940dafe021479749a8eea7ae649f874e9 +Merge: 93637e2135f 0b601c66317 +Author: Tal Levy +Date: Mon Aug 13 09:14:00 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit e122505a9131a9fb23a7923927911bf3058e7636 +Author: Yannick Welsch +Date: Mon Aug 13 18:03:08 2018 +0200 + + Zen2: Deterministic MasterService (#32493) + + Increases testability of MasterService and the discovery layer. Changes: + - Async publish method + - Moved a few interfaces/classes top-level to simplify imports + - Deterministic MasterService implementation for tests + +commit 0b601c663178dcdd9a7361dc193317c3817251eb +Author: Andy Bristol +Date: Thu Jul 12 11:12:37 2018 -0700 + + [test] disable packaging tests for suse boxes + +commit cb2273b02a16477a08933926232b9cca828d6d83 +Author: Nhat Nguyen +Date: Mon Aug 13 10:10:27 2018 -0400 + + Mute IndicesRequestIT#testBulk + + Tracked at #32808 + +commit 4d2144a2a0aa15fe08962172d31b61a8067de8b4 +Author: Dimitris Athanasiou +Date: Mon Aug 13 14:51:13 2018 +0100 + + [ML][DOCS] Refer to rules feature as custom rules (#32785) + +commit d147cd72cc730d543081a9241006cdf2d654800c +Author: Ed Savage <32410745+edsavage@users.noreply.github.com> +Date: Mon Aug 13 10:31:17 2018 +0100 + + [ML] Partition-wise maximum scores (#32748) + + Added infrastructure to push through the 'person name field value' to + the normalizer process. This is required by the normalizer to retrieve + the maximum scores for individual partitions. + +commit 4d20e69b830ffd410a84b2a51f490d97c3c905ca +Author: Morten Segelvik +Date: Mon Aug 13 11:29:18 2018 +0200 + + [DOCS] XContentBuilder#bytes method removed, using BytesReference.bytes(docBuilder) (#32771) + +commit 3e692c3f3d835c91891702c991a4e8881510bdbf +Author: Luca Cavanna +Date: Mon Aug 13 11:05:27 2018 +0200 + + HLRC: migration get assistance API (#32744) + + The request and response classes have been extracted from `IndexUpgradeInfoAction` into top-level classes, and moved to the protocol jar. The `UpgradeActionRequired` enum is also moved. + + Relates to #29827 + +commit 02f2fad57bc95d8402ade6aca4144a2fd1d32861 +Author: Alpar Torok +Date: Mon Aug 13 08:52:16 2018 +0300 + + Add a task to run forbiddenapis using cli (#32076) + + * Add a task to run forbiddenapis using cli + + Add a task that offers an equivalent check to the forbidden APIs plugin, + but runs it using the forbiddenAPIs CLI instead. + + This isn't wired into precommit first, and doesn't work for projects + that require specific signatures, etc. It's meant to show how this can + be used. The next step is to make a custom task type and configure it + based on the project extension from the pugin and make some minor + adjustments to some build scripts as we can't bee 100% compatible with + that at least due to how additional signatures are passed. + + Notes: + - there's no `--target` for the CLI version so we have to pass in + specific bundled signature names + - the cli task already wires to `runtimeJavaHome` + - no equivalent for `failOnUnsupportedJava = false` but doesn't seem to + be a problem. Tested with Java 8 to 11 + - there's no way to pass additional signatures as URL, these will have + to be on the file system, and can't be resources on the cp unless we + rely on how forbiddenapis is implemented and mimic them as boundled + signatures. + - the average of 3 runs is 4% slower using the CLI for :server. + ( `./gradlew clean :server:forbiddenApis` vs `./gradlew clean + :server:forbiddenApisCli`) + - up-to-date checks don't work on the cli task yet, that will happen + with the custom task. + + See also: #31715 + +commit 8114646e12661a860bcc388124eebb74aa8b7ea3 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Sun Aug 12 00:49:08 2018 +1000 + + [Kerberos] Add debug log statement for exceptions (#32663) + + This commit adds missing debug log statements for exceptions + that occur during ticket validation. I thought these + get logged somewhere else in authentication chain + but even after enabling trace logs I could not see them + logged. As the Kerberos exception messages are cryptic + adding full stack trace would help debugging faster. + +commit 38ec0ff6ca483132c54f7dcc0f680de999a3cead +Author: Tim Brooks +Date: Fri Aug 10 17:36:29 2018 -0600 + + Make x-pack core pull transport-nio (#32757) + + The security nio transports depend on transport-nio. This commit + modifies x-pack core to include the transport-nio jar into the x-pack + core module. + +commit 47d2bcc3c4c5224dbda4770a0957fa3a289cc422 +Author: Jack Conradson +Date: Fri Aug 10 14:46:31 2018 -0700 + + Painless: Clean Up Whitelist Names (#32791) + + Renames variables in the whitelists to match the current naming scheme. Mechanical change. + +commit cb1d467124627a547047cbfcc8644f718011b4f9 +Author: Ryan Ernst +Date: Fri Aug 10 13:15:00 2018 -0700 + + Cat apis: Fix index creation time to use strict date format (#32510) + + With the move to java time, the default formatter used by toString on + ZonedDateTime uses optional components for least significant portions of + the date. This commit changes the cat indices api to use a strict date + time format, which will always output milliseconds, even if they are + zero. + + closes #32466 + +commit b08416b899c3350725e12df85b381e8ec80fa2be +Author: Benjamin Trent +Date: Fri Aug 10 13:52:00 2018 -0500 + + Clear Job#finished_time when it is opened (#32605) (#32755) + + * Clear Job#finished_time when it is opened (#32605) + + * not returning failure when Job#finished_time is not reset + + * Changing error log string and source string + +commit f5ba801c6b935097d96f921693d4914fff2049ba +Author: Nik Everett +Date: Fri Aug 10 11:22:23 2018 -0400 + + Test: Only sniff host metadata for node_selectors (#32750) + + Our rest testing framework has support for sniffing the host metadata on + startup and, before this change, it'd sniff that metadata before running + the first test. This prevents running these tests against + elasticsearch installations that won't support sniffing like Elastic + Cloud. This change allows tests to only sniff for metadata when they + encounter a test with a `node_selector`. These selectors are the things + that need the metadata anyway and they are super rare. Tests that use + these won't be able to run against installations that don't support + sniffing but we can just skip them. In the case of Elastic Cloud, these + tests were never going to work against Elastic Cloud anyway. + +commit 93637e2135fe449dbc4addcc871055e5ed2a00c4 +Merge: c7a2c357a30 7a1bbbc5819 +Author: Tal Levy +Date: Fri Aug 10 10:23:14 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 8a2b1a7dcaea9c5aec47852b7e73cecfec9e6d2d +Author: Jonathan Little +Date: Fri Aug 10 10:21:54 2018 -0700 + + Update scripted metric docs to use `state` variable (#32695) + + This change brings the scripted metric agg context docs in line with the new agg state context variable. + +commit 7a1bbbc5819e85a82efb72094dd987bc773e5856 +Author: Jack Conradson +Date: Fri Aug 10 09:55:38 2018 -0700 + + Painless: Clean up PainlessCast (#32754) + + Renames variables and methods in PainlessCast to go along with the new naming + scheme. Mechanical change. + +commit 1dcf80795a197561204116c80ba2657ebbead457 +Author: Albert Zaharovits +Date: Fri Aug 10 19:37:05 2018 +0300 + + [TEST] Certificate NONE not allowed in FIPS JVM (#32753) + + Certificate NONE not allowed when running in a FIPS JVM + +commit c7b1ba33aa217c8624d159c3aa5ac2a0cb6f31b8 +Author: Dimitris Athanasiou +Date: Fri Aug 10 17:28:20 2018 +0100 + + [ML] Refactor ProcessCtrl into Autodetect and Normalizer builders (#32720) + + This moves the helper functionality for creating the autodetect + and mormalizer processes into corresponding builders. + +commit ef21ab8567571104fb578a2627b4380ab4b83efe +Author: Alpar Torok +Date: Fri Aug 10 15:03:17 2018 +0000 + + Access build tools resources (#32201) + + Add a buildResources + + buildResources.take('resource') writes the resources out to the filesystem so the paths can be sent int other tools. + +commit e49717393a268161c0ef0eedf3754b32a9836dff +Author: Alexander Reelsen +Date: Fri Aug 10 16:50:11 2018 +0200 + + Tests: Disable rolling upgrade tests with system key on fips JVM (#32775) + + This disables the x-pack rolling upgrade tests using a fips JVM, as + there are problems creating the keystore. + + Relates #32737 + +commit 2df80ca7e3865e14744aff7852161cb0f82450bf +Author: Nik Everett +Date: Fri Aug 10 10:17:48 2018 -0400 + + HLRC: Ban LoggingDeprecationHandler (#32756) + + LoggingDeprecationHandler requires log4j2 but we don't require log4j2 in + the client. This bans LoggingDeprecationHandler and removes all uses of + it in the high level client. + + Closes #32151 + +commit 22f7b0343046e3baf9dea5b3a8f9b7620f4c649d +Author: Christoph Büscher +Date: Fri Aug 10 15:13:44 2018 +0200 + + Fix test reproducability in AbstractBuilderTestCase setup (#32403) + + Currently AbstractBuilderTestCase generates certain random values in its + `beforeTest()` method annotated with @Before only the first time that a test + method in the suite is run while initializing the serviceHolder that we use for + the rest of the test. This changes the values of subsequent random values + and has the effect that when running single methods from a test suite with + "-Dtests.method=*", the random values it sees are different from when the same + test method is run as part of the whole test suite. This makes it hard to use + the reproduction lines logged on failure. + + This change runs the inialization of the serviceHolder and the randomization + connected to it using the test runners master seed, so reproduction by running + just one method is possible again. + + + Closes #32400 + +commit de2897a1905f16db39f118c2d46e12aaea2434df +Author: Alpar Torok +Date: Fri Aug 10 16:10:13 2018 +0300 + + Only require java_home env var if needed + + We accidentally started enforcing the env vars are always there. + This reverts to only do this if they are requierd. + +commit f236bb3ff68cc5d228951a4a724226e963b52faa +Author: Alexander Reelsen +Date: Fri Aug 10 14:36:41 2018 +0200 + + Tests: Muted ScriptDocValuesDatesTests.testJodaTimeBwc + + Relates #32779 + +commit 94a9b253dbb7c21195d85f55dbf467996d6eaa9f +Author: Benjamin Trent +Date: Fri Aug 10 07:16:55 2018 -0500 + + Adding ML HLRC wrapper and put_job API call (#32726) + + * Adding ML HLRC wrapper and put_job API call + + * Changing integration test job to have consistent stucture + +commit 40b0a3a014b2f53bc555a59b2501f398a3f7446e +Author: Luca Cavanna +Date: Fri Aug 10 14:12:42 2018 +0200 + + [DOCS] Adjust high-level client x-pack docs (#32747) + + - Expose x-pack usage docs page which was not linked in supported-apis page + - make watcher a top-level dir outside of x-pack directory + - move x-pack info and usage pages to miscellaneous + - add new Watcher category to supported-apis (they were under miscellaneous) + - remove x-pack prefix from watcher docs titles + +commit ae0c303dadb401f3df889640ad575aec36a3963e +Author: David Roberts +Date: Fri Aug 10 12:19:06 2018 +0100 + + Move icu4j and super-csv version numbers to versions file (#32769) + + The upcoming ML log structure finder functionality will use these + libraries, and it makes sense to use the same versions that are + being used elsewhere in Elasticsearch. This is especially true + with icu4j, which is pretty big. + +commit 9561a0a66ed4eedd98b479681df81f938234651b +Author: Luca Cavanna +Date: Fri Aug 10 10:55:25 2018 +0200 + + [DOCS] Add missing link to get license API docs (#32746) + +commit 3fa3af1d1166400288103db8546845a52da54a89 +Author: Alexander Reelsen +Date: Fri Aug 10 10:27:42 2018 +0200 + + Tests: Mute test in KerberosRealmTests + + Relates #32701 + +commit f58ed21720d186a3af5ee9eabc1e96a83a0aa6ca +Author: Boaz Leskes +Date: Fri Aug 10 10:15:01 2018 +0200 + + Refactor TransportShardBulkAction to better support retries (#31821) + + Processing bulk request goes item by item. Sometimes during processing, we need to stop execution and wait for a new mapping update to be processed by the node. This is currently achieved by throwing a `RetryOnPrimaryException`, which is caught higher up. When the exception is caught, we wait for the next cluster state to arrive and process the request again. Sadly this is a problem because all operations that were already done until the mapping change was required are applied again and get new sequence numbers. This in turn means that the previously issued sequence numbers are never replicated to the replicas. That causes the local checkpoint of those shards to be stuck and with it all the seq# based infrastructure. + + This commit refactors how we deal with retries with the goal of removing `RetryOnPrimaryException` and `RetryOnReplicaException` (not done yet). It achieves so by introducing a class `BulkPrimaryExecutionContext` that is used the capture the execution state and allows continuing from where the execution stopped. The class also formalizes the steps each item has to go through: + 1) A translation phase for updates + 2) Execution phase (always index/delete) + 3) Waiting for a mapping update to come in, if needed + 4) Requires a retry (for updates and cases where the mapping are still not available after the put mapping call returns) + 5) A finalization phase which allows updates to the index/delete result to an update result. + +commit af8c23eb40b045f4335b592a8a500fe21e84b191 +Author: Alpar Torok +Date: Fri Aug 10 08:07:43 2018 +0000 + + Java version reproduction (#32715) + + Enhance reproduction line with info about jdks + + Provide the ability to control compiler and hava versions just by + passing a property. The actual java home comes from the + `JAVA_HOME` env vars that we allready require. + This works better with the Gradle daemon as well. + + Output is also changed a bit. + + for `-Druntime.java=8 -Dcompiler.java=9`: + ``` + ======================================= + Elasticsearch Build Hamster says Hello! + Gradle Version : 4.9 + OS Info : Linux 4.17.8-1-ARCH (amd64) + Compiler JDK Version : 11 (Oracle Corporation 11-ea [OpenJDK 64-Bit Server VM 11-ea+22]) + Runtime JDK Version : 11 (Oracle Corporation 11-ea [OpenJDK 64-Bit Server VM 11-ea+22]) + Gradle JDK Version : 10 (Oracle Corporation 10.0.1 [OpenJDK 64-Bit Server VM 10.0.1+10]) + Compiler java.home : /home/alpar/opt/jdk-11-ea22/ + Runtime java.home : /home/alpar/opt/jdk-11-ea22/ + Gradle java.home : /usr/lib/jvm/java-10-openjdk + Random Testing Seed : EA858533191E8DFB + ======================================= + ``` + + Without configuration: + ``` + ======================================= + Elasticsearch Build Hamster says Hello! + ======================================= + Gradle Version : 4.9 + OS Info : Linux 4.17.8-1-ARCH (amd64) + JDK Version : 10 (Oracle Corporation 10.0.1 [OpenJDK 64-Bit Server VM 10.0.1+10]) + JAVA_HOME : /usr/lib/jvm/java-10-openjdk + Random Testing Seed : 4BD5B2A839C8FCA1 + ======================================= + ``` + + Here's how a reproduction line will look like (test made to fail): + ``` + ./gradlew :modules:lang-painless:test -Dtests.seed=2DA2379065A4EEAB -Dtests.class=org.elasticsearch.painless.AdditionTests -Dtests.method="testInt" -Dtests.security.manager=true -Dtests.locale=es-PE -Dtests.timezone=WET -Dcompiler.java=10 -Druntime.java=10 + ``` + +commit 798fb546cbff3f20d562054c2822b752a69d7c1d +Author: Alexander Reelsen +Date: Fri Aug 10 09:38:18 2018 +0200 + + Core: Create java time based DateMathParser (#32131) + + This adds a java time based date math parser class in order, which will replace the joda date based one in the future. For now the class also returns the date in milliseconds since the epoch. + +commit 93d4f84b215d42bd728c447594a3b6d0b6af1c29 +Author: Tim Vernum +Date: Fri Aug 10 13:36:36 2018 +1000 + + [TEST] Add "ne" as an unsupported SimpleKdc locale (#32700) + + SimpleKdcServer cannot handle some locales, so we explicitly avoid + using them in tests. + Adding "ne" (Nepali) to that list. + +commit c7a2c357a308eb694794282d309323ed0908f632 +Merge: 41e6d98af80 0161bbc77f2 +Author: Tal Levy +Date: Thu Aug 9 18:00:40 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 0161bbc77f2030bf95346324d2fe9fc6e41aa9e3 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Aug 10 09:55:43 2018 +1000 + + [Test] Handle file permissions for Windows (#32681) + + This commit modifies the test to handle file permission + tests in windows/dos environments. The test requires access + to UserPrincipal and so have modified the plugin-security policy + to access user information. + + Closes#32637 + +commit 41e6d98af80cc42a3fe18b025719b6a3d65d1e5c +Author: Tal Levy +Date: Thu Aug 9 16:09:20 2018 -0700 + + move qa yaml tests to inside the ILM plugin (#32693) + + The qa tests with security haven't actually gone as far as testing security roles yet, so this is a start in the hopes of both bringing the tests into the ilm plugin + +commit 293c8a2b24a611e1f8de77d1b46fd041bf1e386c +Author: Jack Conradson +Date: Thu Aug 9 14:24:55 2018 -0700 + + Painless: Add an Ingest Script Processor Example (#32302) + + This commit adds two pieces. The first is a small set of documentation providing + instructions on how to get setup to run context examples. This will require a download + similar to how Kibana works for some of the examples. The second is an ingest processor + example using the downloaded data. More examples will follow as ideally one per PR. + This also adds a set of tests to individually test each script as a unit test. + +commit be54ba39c4042419dbbbfa149d92aaf3396475cd +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Thu Aug 9 23:10:51 2018 +0200 + + Add expected mapping type to `MapperException` (#31564) + + Currently if a document cannot be indexed because it violates the defined + mapping for the index, a MapperException is thrown. In some cases it is + useful to expose the expected field type in the exception itself, + so that the user can react based on the error message. This change adds + the expected data type to the MapperException. + + Closes #31502 + +commit 294ab7ee96267549af8c5b15d8493c6c7fb49ea2 +Author: Nik Everett +Date: Thu Aug 9 16:11:48 2018 -0400 + + Core: Remove some logging constructors (#32513) + + Remove a few of the logger constructors that aren't widely used or + aren't used at all and deprecate a few more logger constructors in favor + of log4j2's `LogManager`. + +commit 8750d622fcf2255b9abd5ac50770b43c27834ed1 +Author: Colin Goodheart-Smithe +Date: Thu Aug 9 20:39:06 2018 +0100 + + Adds REST client support for starting and stopping ILM (#32609) + + * Adds REST client support for PutOperationMode in ILM + + * Corrects licence headers + + * iter + + * add request converter test + + * Fixes tests + + * Creates start and stop actions for controlling ILM operation + + * Addresses review comments + +commit 5bbed5ed9a475a31822c385228af00b5d0507be9 +Merge: c5e66cb3451 abda9fdac1f +Author: Alpar Torok +Date: Thu Aug 9 21:58:03 2018 +0300 + + Merge branch 'master' of github.com:elastic/elasticsearch + +commit c5e66cb3451510c9e9e53736e14b04bb58feebd1 +Author: Alpar Torok +Date: Thu Aug 9 21:54:58 2018 +0300 + + mute test #32737 + +commit abda9fdac1f8e300e73f1ba22d8e336831f35c4b +Author: Nik Everett +Date: Thu Aug 9 14:35:30 2018 -0400 + + Logging: Fix test on windows + + Windows' `\` instead of `/` strikes again! + + Closes #32546 + +commit 9a16491ebf58e0c0b7dad3dcf51690367cb5202e (es/v6.3) +Author: Ryan Ernst +Date: Thu Aug 9 10:11:08 2018 -0700 + + Build: Add back setting artifact id of pom for rest high level client (#32731) + + This commit adds back the publishing section that sets the artifact id + of the generated pom file for the high level rest client. This was + accidentally removed during a consolidationo of the shadow plugin logic. + +commit 2305cb8a6b1f2388ed00f089fa808f590058b962 +Author: Colin Goodheart-Smithe +Date: Thu Aug 9 17:45:29 2018 +0100 + + Fixes test after replicas removal + +commit 7b618f3db4148e2205b3d88b35adc2c38d6e694e +Author: Andrei Stefan +Date: Thu Aug 9 19:30:02 2018 +0300 + + SQL: Bug fix for the optional "start" parameter usage inside LOCATE function (#32576) + + The incorrect NodeInfo is created when the optional parameter is not used, leading to the incorrect constructor being used. Simplified LocateFunctionProcessorDefinition by using one constructor instead of two. + Fixes https://github.com/elastic/elasticsearch/issues/32554 + +commit 6750e157737a1f17260622f05660793647b9f017 +Author: Andrei Stefan +Date: Thu Aug 9 19:28:48 2018 +0300 + + SQL: Ignore H2 comparative tests for uppercasing/lowercasing string functions (#32604) + + Skip the comparative tests using lowercasing/uppercasing against H2 (which considers the Locale). + ES-SQL is, so far, ignoring the Locale. + Still, the same queries are executed against ES-SQL alone and results asserted to be correct. + +commit e162127ff3cef81f23c56176b756217b5d63dac1 +Author: Nicholas Knize +Date: Wed Aug 8 17:11:33 2018 -0500 + + Upgrade to Lucene-7.5.0-snapshot-13b9e28f9d + + The main feature is the inclusion of bkd backed geo_shape with + INTERSECT, DISJOINT, WITHIN bounding box and polygon query support. + +commit 79375d35bbfdfcd380049cbc59d259fc31102d00 +Author: Armin Braun +Date: Thu Aug 9 14:32:36 2018 +0200 + + Scripting: Replace Update Context (#32096) + + * SCRIPTING: Move Update Scripts to their own context + * Added system property for backwards compatibility of change to `ctx.params` + +commit 0fe21136dbeea2343a516ac84e632ecafc5c6176 +Merge: 97ba7b83b44 823d40e19b3 +Author: Colin Goodheart-Smithe +Date: Thu Aug 9 12:47:26 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 97ba7b83b447b085d64edfb2eb7960254b2a1d0d +Author: Colin Goodheart-Smithe +Date: Thu Aug 9 12:24:35 2018 +0100 + + Fix explain lifecycle REST API test failure + +commit 5ff4f9347fe37eb5b72d444c01ebb0cbc9f17bbc +Author: Colin Goodheart-Smithe +Date: Thu Aug 9 10:18:45 2018 +0100 + + Adds explain lifecycle API to the Rest Client (#32606) + +commit 823d40e19b3cbc2b314b49b63d4666caf70059ae +Author: Alexander Reelsen +Date: Thu Aug 9 10:01:40 2018 +0200 + + Core: Fix Java Time DateFormatter printers (#32592) + + A bug in the test suite prevented to properly check that all date + formatters printed the date the same way like joda time does. + + This fixes the test and thus also a fair share of formats, that + now use the strict parser for printing. + +commit 9b00f095b94c7fb6b568e5e36a264d2ae8c293c5 +Author: Jack Conradson +Date: Wed Aug 8 16:25:14 2018 -0700 + + Painless: Move More Logic to PainlessLookup (#32689) + + This moves some run-time lookups for methods and fields to the PainlessLookup. + +commit 99c3d8ab26f3fad2e5bcbd59a407493229d8c65b +Author: Julie Tibshirani +Date: Wed Aug 8 14:22:03 2018 -0700 + + Fix a bug in the BWC tests where we added a field alias in unsupported versions. + +commit 7af28c48c364e6f3b094a58411f2e244babd6d85 +Author: Lee Hinman +Date: Wed Aug 8 16:21:58 2018 -0600 + + Switch WritePipelineResponse to AcknowledgedResponse (#32722) + + We previously discussed moving the classes extending `AcknowledgedResponse` to + simply use `AcknowledgedResponse`, making the class non-abstract. + + This moves the first class to do this, removing `WritePipelineResponse` in the + process. + + If we like the way this looks, I will switch the remaining classes over to using + `AcknowledgedResponse`. + +commit 433b7b8427c4c7f9bbe943a3a5433e69bad8c3f7 +Author: David Turner +Date: Wed Aug 8 21:44:44 2018 +0100 + + Remove FutureExecutor interface (#32713) + + Instead of using a separate interface for scheduling tasks in the future, we + can simply use ThreadPool#schedule with a suitable implementation. This removes + the unnecessary interface and its usages, and migrates the tests as + appropriate. + +commit 2fc3f1d04ce8f1aa78ac8f71dbfb8a7180010b79 +Author: Tal Levy +Date: Wed Aug 8 11:43:29 2018 -0700 + + move replicas action functionality into AllocateAction (#32523) + + Since replica counts and allocation rules are set separately, it is not always clear how many replicas are to be allocated in the allocate action. Moving the replicas action to occur at the same time as the allocate action, resolves this confusion that could end an undesired state. This means that the ReplicasAction is removed, and a new optional replicas parameter is added to AllocateAction. + +commit d586e4cfd3edbebdad70ac94a64a9a54278b0d36 +Author: Benjamin Trent +Date: Wed Aug 8 11:54:11 2018 -0500 + + Adding `Job` and `AnalysisConfig` for HLRC (#32687) + + * Adding `Job` and `AnalysisConfig` for HLRC + + * Removing println used for local debugging + + * Adding null checks and removing unnecessary field + +commit f1869cca359bb00b156b87360271b10be37edc13 +Author: Jim Ferenczi +Date: Wed Aug 8 16:36:51 2018 +0200 + + Fix role query that can match nested documents (#32705) + + This change makes sure that the role query excludes nested documents + when it is ran to select the parent documents allowed by the role. + +commit 7fdf898518a9e2418e0e88583cfd2f8f025ec1f6 +Author: Suresh N S <41610499+nssuresh2007@users.noreply.github.com> +Date: Wed Aug 8 19:54:53 2018 +0530 + + Whitelisting / from Circuit Breaker Exception (#32325) (#32666) + + When Circuit Breaker has tripped, certain diagnostic requests like + "_cluster/health" succeed where as request to / fails with + 503 Service Unavailable. This behavior is observed because of this + commit f32b700 where certain API paths are whitelisted from + Circuit Breaking exception, but / is not whitelisted. + Added / to circuit breaker whitelist so that it can be used for + diagnostic purposes + +commit 2d925c9a9a3b05faa988335aa230b3071fca7718 +Merge: b9c04adb295 7d641ba69b1 +Author: Tal Levy +Date: Wed Aug 8 07:21:01 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 7d641ba69b15cb3f5e9aefe8aab4c4d833a43602 +Author: Armin Braun +Date: Wed Aug 8 15:47:51 2018 +0200 + + TESTS: Explicitly Fail Http Client Timeouts (#32708) + + * Don't quietly ignore timeouts when waiting for HTTP responses + * Fixes #32702 + +commit 781e6ad5517affe4adb769ccab9bf1c9ff0798b6 +Author: Colin Goodheart-Smithe +Date: Wed Aug 8 12:46:38 2018 +0100 + + Fixes suggestion generics (#32706) + + * Fixes suggestion generics + + This solves a compile problem in Eclipse where Eclipse could not + resolve the generics for the options field in `PhraseSuggestion.Entry`. + But I think this is also a good change in general because + `PhraseSuggestion.Entry` is now declaring the specific `Option` + implementation it requires rather than `Suggest.Entry.Option` which is + more general and could lead to weird bugs. `CompletionSuggestion.Entry` + and `TermSuggestion.Entry` already declare the more specific class they + use so I think this was an oversight in `PhaseSuggestion.Entry` + + * iter + +commit 580d59e2d75b28437764af8c5153e9e91bedac97 +Author: Armin Braun +Date: Wed Aug 8 12:04:25 2018 +0200 + + CORE: Upgrade to Jackson 2.8.11 (#32670) + + * closes #30352 + +commit 3e437438d542e81d534d9e6c4463734e65470b83 +Author: Luca Cavanna +Date: Wed Aug 8 09:59:22 2018 +0200 + + Prevent cause from being null in ShardOperationFailedException (#32640) + + `ShardOperationFailedException` and corresponding implementors seem to suggest that the cause may be null, case that is also handled in a few places. Yet, it does not seem to be possible in practice for the cause to be null, hence we can clean that up and enforce the cause to be a non null value. This is best done by making `ShardOperationFailedException` an abstract class rather than an interface, which holds the basic member instance that all the subclasses have in common and can also enforce that cause, status and reason are non null. + +commit 5c2ef5e86908f82bced8bec6be0a193453ee276b +Author: Luca Cavanna +Date: Wed Aug 8 09:57:11 2018 +0200 + + Preserve index_uuid when creating QueryShardException (#32677) + + As part of #32608 we made sure that the fully qualified index name is taken from the query shard context whenever creating a new `QueryShardException`. That change introduced a regression as instead of setting the entire `Index` object to the exception, which holds index name and index uuid, we ended up setting only the index name (including cluster alias). With this commit we make sure that the index uuid does not get lost and we try to lower the chances that a similar bug makes it in another time. That's done by making `QueryShardContext` return the fully qualified `Index` (which also holds the uuid) rather than only the fully qualified index name. + +commit 4cbcc1d659c7d51703345cce01f51fb24f4e9bde +Author: Tim Vernum +Date: Wed Aug 8 13:47:18 2018 +1000 + + [TEST] Add empty test case for FIPS + + All other tests in security:cli are skipped on FIPS JVMs + +commit 6a7d5bce8ad240c5134ade56d8937dd267c01550 +Author: Tim Brooks +Date: Tue Aug 7 18:04:34 2018 -0600 + + Use nio driven http client for testing (#32621) + + This is related to #27260. This commit replaces the netty driven http + client (Netty4HttpClient) with one that is driven by (NioHttpClient). + This client exists in the test package and is used for making http + requests. + +commit 0281846c6b82135acfdaa97a5a1de496320badfe +Author: Julie Tibshirani +Date: Tue Aug 7 16:48:41 2018 -0700 + + Adjust the test version now that the field alias bug has been backported. + +commit d7183f8f3ddaaa921fec1c29cc7093ca59c104e4 +Author: Julie Tibshirani +Date: Tue Aug 7 16:20:09 2018 -0700 + + Make sure that field collapsing supports field aliases. (#32648) + +commit 8bfb0f3f8d9e9032fdc10ce3fcd1473933a59019 +Author: Andy Bristol +Date: Tue Aug 7 13:31:00 2018 -0700 + + serialize suggestion responses as named writeables (#30284) + + Suggestion responses were previously serialized as streamables which + made writing suggesters in plugins with custom suggestion response types + impossible. This commit makes them serialized as named writeables and + provides a facility for registering a reader for suggestion responses + when registering a suggester. + + This also makes Suggestion responses abstract, requiring a suggester + implementation to provide its own types. Suggesters which do not need + anything additional to what is defined in Suggest.Suggestion should + provide a minimal subclass. + + The existing plugin suggester integration tests are removed and + replaced with an equivalent implementation as an example + plugin. + +commit 0b7fb4e7b943c4e48d8dde7a5be0839231062fdf +Author: Jack Conradson +Date: Tue Aug 7 12:26:57 2018 -0700 + + Painless: Clean up FunctionRef (#32644) + + This change consolidates all the logic for generating a FunctionReference (renamed from + FunctionRef) from several arbitrary constructors to a single static function that is used at + both compile-time and run-time. This increases long-term maintainability as it is much + easier to follow when and how a function reference is being generated. It moves most of + the duplicated logic out of the ECapturingFuncRef, EFuncRef and ELambda nodes and + Def as well. + +commit dcc816427ef62a60690d80ff14f4caec4b471c72 +Author: Jason Tedor +Date: Tue Aug 7 15:10:09 2018 -0400 + + Expose whether or not the global checkpoint updated (#32659) + + It will be useful for future efforts to know if the global checkpoint + was updated. To this end, we need to expose whether or not the global + checkpoint was updated when the state of the replication tracker + updates. For this, we add to the tracker a callback that is invoked + whenever the global checkpoint is updated. For primaries this will be + invoked when the computed global checkpoint is updated based on state + changes to the tracker. For replicas this will be invoked when the local + knowledge of the global checkpoint is advanced from the primary. + +commit 3d5e9114e38af11324f6117947aa270da0d1d691 +Author: Tim Brooks +Date: Tue Aug 7 12:52:28 2018 -0600 + + Reduce connections used by MockNioTransport (#32620) + + The MockNioTransport (similar to the MockTcpTransport) is used for integ + tests. The MockTcpTransport has always only opened a single for all of + its work. The MockNioTransport has awlays opened the default number of + connections (13). This means that every test where two transports + connect requires 26 connections. This is more than is necessary. This + commit modifies the MockNioTransport to only require 3 connections. + +commit 2e65bac5ddb5d5fe5714d9288ef31deee8a663f7 +Author: Tanguy Leroux +Date: Tue Aug 7 18:54:42 2018 +0200 + + [Rollup] Remove builders from RollupJobConfig (#32669) + +commit 733f84c49bf0161164fbe64b087046c4080b4438 +Author: David Roberts +Date: Tue Aug 7 17:49:48 2018 +0100 + + [DOCS] Add missing docs include + +commit 2608012422b7bb3621cdcc9f12736139c1ae9bdb +Author: David Roberts +Date: Tue Aug 7 16:59:56 2018 +0100 + + Add temporary directory cleanup workarounds (#32615) + + On some Linux distributions tmpfiles.d cleans files and + directories under /tmp if they haven't been accessed for + 10 days. + + This can cause problems for ML as ML is currently the only + component that uses the temp directory more than a few + seconds after startup. If you didn't open an ML job for + 10 days and then tried to open one then the temp directory + would have been deleted. + + This commit prevents the problem occurring in the case of + Elasticsearch being managed by systemd, as systemd private + temp directories are not subject to periodic cleanup (by + default). + + Additionally there are now some docs to warn people about + the risk and suggest a manual mitigation for .tar.gz users. + +commit 6d50d8b5a91b971af5f20235ac580fa16a0935a0 +Author: Benjamin Trent +Date: Tue Aug 7 10:51:52 2018 -0500 + + Adding job process pojos to protocol pkg (#32657) + + * Adding job process pojos to protocol pkg + + * Removing unused `RESULTS_FIELD` + + * Addressing PR comments, removing unnecessary methods + +commit b3e15851a2830e93b0e34924592e6978057374bd +Author: Lee Hinman +Date: Tue Aug 7 09:36:37 2018 -0600 + + [TEST] Comment out account breaker assertion while diagnosing + + Relates to #30290 + +commit 22c367315ab27638e13e5e48558184e30953180f +Author: Yannick Welsch +Date: Tue Aug 7 16:37:55 2018 +0200 + + [Zen2] Randomized testing of CoordinationState (#32242) + + Simulates a random run of a cluster with multiple CoordinationState instances (each representing + one node), passing messages back and forth, and asserting that the overall system satisfies a given + set of safety properties. + + Follow-up to #32171 + +commit 45066b5e8939c78be7b4c36a691fa90dd5766c8c +Author: Yannick Welsch +Date: Tue Aug 7 15:02:37 2018 +0200 + + Verify primary mode usage with assertions (#32667) + + Primary terms were introduced as part of the sequence-number effort (#10708) and added in ES + 5.0. Subsequent work introduced the replication tracker which lets the primary own its replication + group (#25692) to coordinate recovery and replication. The replication tracker explicitly exposes + whether it is operating in primary mode or replica mode, independent of the ShardRouting object + that's associated with a shard. During a primary relocation, for example, the primary mode is + transferred between the primary relocation source and the primary relocation target. After + transferring this so-called primary context, the old primary becomes a replication target and the + new primary the replication source, reflected in the replication tracker on both nodes. With the + most recent PR in this area (#32442), we finally have a clean transition between a shard that's + operating as a primary and issuing sequence numbers and a shard that's serving as a replication + target. The transition from one state to the other is enforced through the operation-permit system, + where we block permit acquisition during such changes and perform the transition under this + operation block, ensuring that there are no operations in progress while the transition is being + performed. This finally allows us to turn the best-effort checks that were put in place to prevent + shards from being used in the wrong way (i.e. primary as replica, or replica as primary) into hard + assertions, making it easier to catch any bugs in this area. + +commit 3ce984d74690d47618ee53f19d43e2ef69601dbb +Author: Paul Sanwald +Date: Tue Aug 7 08:56:00 2018 -0400 + + mute test while I work on #32215 + +commit 785b6e824c8fe00e81c0f6a1548858d213c2ccf4 +Author: Yannick Welsch +Date: Tue Aug 7 14:51:46 2018 +0200 + + Zen2: Cluster state publication pipeline (#32584) + + Implements the state machine on the master to publish a cluster state. + + Relates to #32006 + +commit f44ba04aeefb02ec9e2993812926afe76a914b38 +Author: David Turner +Date: Tue Aug 7 13:34:53 2018 +0100 + + [Zen2] Add UnicastConfiguredHostsResolver (#32642) + + The `PeerFinder`, introduced in #32246, obtains the collection of seed + addresses configured by the user from a `ConfiguredHostsResolver`. In reality + this collection comes from the `UnicastHostsProvider` via a slightly + complicated threading model that performs the resolution of hostnames to + addresses using a dedicated `ExecutorService`. This commit introduces an + adapter to allow the `PeerFinder` to obtain its seed addresses in this manner. + +commit 289e34aeedc185e83ea03b39b9c59666d849b0bb +Author: David Turner +Date: Tue Aug 7 13:34:07 2018 +0100 + + [Zen2] Add HandshakingTransportAddressConnector (#32643) + + The `PeerFinder`, introduced in #32246, needs to be able to identify, and + connect to, a remote master node using only its `TransportAddress`. This can be + done by opening a single-channel connection to the address, performing a + handshake, and only then forming a full-blown connection to the node. This + change implements this logic. + +commit b9c04adb295ef9c6e27d6bc76521f5a2f0a5703b +Merge: 0ad252d502a 6449d9bc143 +Author: Colin Goodheart-Smithe +Date: Tue Aug 7 12:35:32 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 6449d9bc1438ce00d8e34aee4589228301def059 +Author: Andrey Ershov +Date: Tue Aug 7 13:03:43 2018 +0200 + + Include translog path in error message when translog is corrupted (#32251) + + Currently, when TranslogCorruptedException is thrown most of the times it does not contain information about the translog location on the file system. There is the translog recovery tool that accepts the translog path as an argument and users are constantly puzzled where to get the path. + This pull request adds "source" information to every TranslogCorruptedException thrown. The source could be local file, remote translog source (used for recovery), assertion (translog entry is constructed to perform some assertion) or translog constructed inside the test. + Closes #24929 + +commit 1f50950099fa156269d56bd78378f6732527059b +Author: Albert Zaharovits +Date: Tue Aug 7 13:21:39 2018 +0300 + + Add @AwaitsFix for #32673 + +commit 6fe6247dc80ebf3da6fdd522bff779d73b1d192f +Author: Parth Verma +Date: Tue Aug 7 14:26:44 2018 +0530 + + Ignore script fields when size is 0 (#31917) + + This change adds a check so that when parsing the search source, script fields are + ignored when the requested search result size is 0. This helps with e.g. clients like + Kibana that sends a list of script fields that they may need for convenience, but they + don't require any hits. Before this change, user sometimes ran into confusing behaviour, + e.g. the script compilation limit to breaking although no hits were requested. + + Closes #31824 + +commit ab81078949fadd519a0a36cb5cf9b45ba05b2f85 +Author: simonzheng +Date: Tue Aug 7 04:33:46 2018 -0400 + + [Docs] Correct a small typo (#32655) + +commit f57cb10d2c6f3a99bb54945fe7b8a55a255473da +Author: Armin Braun +Date: Tue Aug 7 10:30:45 2018 +0200 + + Tests: Fix Typo Causing Flaky Settings Test (#32665) + + * We were comparing the wrong timeout value in the `randomValueOtherThan` call here, leading to no mutation happening for a certain seed + * closes #32639 + +commit 1122314b3bdd803f72ef93f6930522995a381eba +Author: Tanguy Leroux +Date: Tue Aug 7 09:39:24 2018 +0200 + + [Rollup] Remove builders from GroupConfig (#32614) + +commit 3fb09231826f4f5c63760123e77a587ee63fa9fb +Author: Jason Tedor +Date: Mon Aug 6 18:07:46 2018 -0400 + + Fix content type detection with leading whitespace (#32632) + + Today content type detection on an input stream works by peeking up to + twenty bytes into the stream. If the stream is headed by more whitespace + than twenty bytes, we might fail to detect the content type. We should + be ignoring this whitespace before attempting to detect the content + type. This commit does that by ignoring all leading whitespace in an + input stream before attempting to guess the content type. + +commit b46e13629f4c02e5abbfcad16175b91a3e2d8279 +Author: Jack Conradson +Date: Mon Aug 6 14:43:53 2018 -0700 + + Docs: Allow snippets to have line continuation (#32649) + + Currently, snippets in lists cannot be rendered correctly as a console command because the console command requires a line continuation '+'. This allows snippets to have a line continuation between the snippet and the // CONSOLE. + +commit 0ad252d502a6b6bb00b6be6e9e0838445688288b +Author: Tal Levy +Date: Mon Aug 6 14:41:27 2018 -0700 + + change default indices.lifecycle.poll_interval to something sane (#32521) + + This was originally set to a few seconds while prototyping things. + This interval is for the scheduled trigger of policies. Policies + have this extra trigger beyond just on cluster-state changes because + cluster-state changes may not be happeneing in a cluster for + whatever reason, and we need to continue making progress. Updating + this value to be larger is reasonable since not all operations + are expected to be completed in the span of seconds, but instead in + minutes and hours. 10 minutes is sane. + +commit 4dda5a990bbf78c73569cbb27c8a47afcadcae27 +Author: Armin Braun +Date: Mon Aug 6 22:46:26 2018 +0200 + + INGEST: Fix ThreadWatchDog Throwing on Shutdown (#32578) + + * INGEST: Fix ThreadWatchDog Throwing on Shutdown + + * #32539 is caused by the fact that ThreadWatchDog.Default could throw on shutdown if the ThreadPool is interrupted while `interruptLongRunningExecutions` is in progress. This is a result of the watchdog not having a lifecycle of its own (normally it terminates when the threadpool terminates). + * We can't easily use `org.elasticsearch.common.util.concurrent.EsRejectedExecutionException#isExecutorShutdown` to catch this state the same way other components do since thatwould require adding the core lib to Grok as a dependency + * Since we have no knowledge of the lifecycle in this compontent since we're only passed the scheduler `BiFunction` I fixed this by only scheduling the watchdog when there's actually registered threads in it. + * I think using the patter of locking via two `Atomic*` values should not be much of a performance concern here under load since either the integer will likely be > 0 in this case (because we have multiple Grok in parallel) or the running state will be true because there likely was at least one thread registered when the watchdog ran and so the enqueing of the watchdog task during `register` will happen very rarely here (in the worst case scenario of only a single Grok thread it will happen less frequently than once every `ingest.grok.watchdog.interval`). The atomic update on the count should not be relevant relative to the cost of adding a new node to the CHM either. + * Fixes #32539 + * Also fixes the watchdog to run if it doens't have to in general. + +commit b2a0f38a0c9f0fd9127e63e22a94f253b7b6d9e5 +Author: Benjamin Trent +Date: Mon Aug 6 15:33:18 2018 -0500 + + Adding xpack.core.ml.datafeed to protocol.xpack.ml.datafeed (#32625) + + * Adding org.elasticsearch.xpack.core.ml.datafeed to org.elasticsearch.protocol.xpack.ml.datafeed + + * removing unused ParseField and import + + * Addressing PR feed back and fixing tests + + * Simplifying Datafeed(Config|Update) ctor parser + +commit e01e4393a8f867d4a23b8a3d6f999e542439759d +Author: DeDe Morton +Date: Mon Aug 6 12:00:07 2018 -0700 + + [Docs] Light edit to info about docker images (#32376) + +commit 919888eba7b30f5d06364c2dbb8f61b0b1caf301 (es/ccr) +Author: Nhat Nguyen +Date: Mon Aug 6 14:55:23 2018 -0400 + + TEST: Enable debug log testValidateFollowingIndexSettings + +commit 014b2772db003635bcb971df40e53d81e1549062 +Author: Yannick Welsch +Date: Mon Aug 6 18:35:18 2018 +0200 + + [TEST] Fix testReplicaTermIncrementWithConcurrentPrimaryPromotion + + The assertion in the test was not broad enough. If the timing is very unlucky, the + shard is already promoted to primary before the indexOnReplica even gets to execute. + + Closes #32645 + +commit c394eb9ae98626c9c6c6e4c60f829af4e7b4ade4 +Author: Nhat Nguyen +Date: Mon Aug 6 10:55:37 2018 -0400 + + CCR: Expose the operation primary term + + Relates #32442 + +commit 0a9c3ae8bcc040e4586b98a0cef5d1c434aab4f9 +Author: Lee Hinman +Date: Mon Aug 6 08:53:44 2018 -0600 + + Remove UpdateSettingsTestHelper class (#32557) + + * Remove UpdateSettingsTestHelper class + + By making the `settings()` method public on `UpdateSettingsRequest` (I think it + should have been in the first place) we can get rid of this class entirely. Mock + response objects are now constructed by parsing JSON without making the + constructor public. + + Relates to #29823 + +commit 7ea7dd8018ce6d6bc351662e3e0276d3982c52d4 +Author: Lee Hinman +Date: Mon Aug 6 08:53:28 2018 -0600 + + Remove RolloverIndexTestHelper (#32559) + + * Remove RolloverIndexTestHelper + + This removes the `RolloverIndexTestHelper` class in favor of making a couple of + getters publically accessible as well as custom building a response object using + JSON parsing. + + Relates to #29823 + +commit aed466d5b6cf7e7e5f4f5ad63eccbdd8cc892d1b +Author: Lee Hinman +Date: Mon Aug 6 08:53:12 2018 -0600 + + Remove ILM constructor hacks (#32597) + + This commit removes the hacks associated with mocking Response objects. Rather + than parse a wrapped byte array, the constructors for `IndicesAliasesResponse` + and `ResizeResponse` are made public + + Relates to #29823 + +commit 5881322b3f01f61cb810235fd1f8b475baef0c9c +Merge: 3b739b9fd5d 826399f9fce +Author: Nhat Nguyen +Date: Mon Aug 6 10:27:18 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Cross-cluster search: preserve cluster alias in shard failures (#32608) + Handle AlreadyClosedException when bumping primary term + [TEST] Allow to run in FIPS JVM (#32607) + [Test] Add ckb to the list of unsupported languages (#32611) + SCRIPTING: Move Aggregation Scripts to their own context (#32068) + Painless: Use LocalMethod Map For Lookup at Runtime (#32599) + [TEST] Enhance failure message when bulk updates have failures + [ML] Add ML result classes to protocol library (#32587) + Suppress LicensingDocumentationIT.testPutLicense in release builds (#32613) + [Rollup] Update wire version check after backport + Suppress Wildfly test in FIPS JVMs (#32543) + [Rollup] Improve ID scheme for rollup documents (#32558) + ingest: doc: move Dot Expander Processor doc to correct position (#31743) + [ML] Add some ML config classes to protocol library (#32502) + [TEST]Split transport verification mode none tests (#32488) + Core: Move helper date formatters over to java time (#32504) + [Rollup] Remove builders from DateHistogramGroupConfig (#32555) + [TEST} unmutes SearchAsyncActionTests and adds debugging info + [ML] Add Detector config classes to protocol library (#32495) + [Rollup] Remove builders from MetricConfig (#32536) + Tests: Add rolling upgrade tests for watcher (#32428) + Fix race between replica reset and primary promotion (#32442) + +commit 2176184db18ddcae22d3506c38d35a08c86fb4b4 +Author: David Turner +Date: Mon Aug 6 15:26:31 2018 +0100 + + [Zen2] Introduce gossip-like discovery of master nodes (#32246) + + This commit introduces the `PeerFinder` which can be used to collect the + identities of the master-eligible nodes in a masterless cluster, based on the + `UnicastHostsProvider`, the nodes in the `ClusterState`, and nodes that other + nodes have discovered. + +commit e641fccfe3e7ce7ee9d41235cf29800d440e0d26 +Author: Igor Motov +Date: Mon Aug 6 07:15:40 2018 -0700 + + Rest HL client: Add get license action (#32438) + + Rest HL client: Add get license action + + Continues to use String instead of a more complex License class to + hold the license text similarly to put license. + + Relates #29827 + +commit 615aa85f4eb667649cb053c384d62ffb65e13f57 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Mon Aug 6 23:51:43 2018 +1000 + + [Kerberos] Use canonical host name (#32588) + + The Apache Http components support for Spnego scheme + uses canonical name by default. + Also when resolving host name, on centos by default + there are other aliases so adding them to the + DelegationPermission. + + Closes#32498 + +commit 0a67cb413333b2fa3f53ee2fc54408767e3b66c6 +Author: Armin Braun +Date: Mon Aug 6 14:56:21 2018 +0200 + + LOGGING: Upgrade to Log4J 2.11.1 (#32616) + + * LOGGING: Upgrade to Log4J 2.11.1 + * Upgrade to `2.11.1` to fix memory leaks in slow logger when logging large requests + * This was caused by a bug in Log4J https://issues.apache.org/jira/browse/LOG4J2-2269 and is fixed in `2.11.1` via https://git-wip-us.apache.org/repos/asf?p=logging-log4j2.git;h=9496c0c + * Fixes #32537 + * Fixes #27300 + +commit 3b739b9fd5da2323a84b242708b64c16660113cc +Author: Jason Tedor +Date: Mon Aug 6 08:01:47 2018 -0400 + + Avoid NPE on shard changes action (#32630) + + If a leader index is deleted while there is an active follower, the + follower will send shard changes requests bound for the leader + index. Today this will result in a null pointer exception because there + will not be an index routing table for the index. A null pointer + exception looks like a bug to a user so this commit addresses this by + throwing an index not found exception instead. + +commit 826399f9fce7792b5a5fbb9456599144b4d6bd23 +Author: Luca Cavanna +Date: Mon Aug 6 11:48:50 2018 +0200 + + Cross-cluster search: preserve cluster alias in shard failures (#32608) + + When some remote clusters return shard failures as part of a cross-cluster search request, the cluster alias currently gets lost. As a result, if the shard failures are all caused by the same error, and against indices belonging to different clusters, but with the same index name, only one failure gets returned as part of the search response, meaning that failures are grouped by index name, ignoring the cluster alias. + + With this commit we make sure that `ShardSearchFailure` returns the cluster alias as part of the index name. Also, we set the fully qualfied index name when creating a `QueryShardException`. That way shard failures are grouped by cluster:index. Such fixes should cover at least most of the cases where either 1) the shard target is set but we don't have the index in the cause (we were previously reading it only from the cause that did not have the cluster alias) 2) the shard target is missing but if the cause is a `QueryShardException` the cluster alias does not get lost. + + We also prevent NPE in case the failure cause is not set and test such scenario. + +commit c13b85d6d38a303e3506667a2b3e732912a2f1de +Merge: 457661e0dab 3cf08326ab0 +Author: Colin Goodheart-Smithe +Date: Mon Aug 6 10:31:33 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 3cf08326ab025aa7f9ac81e70c38313c625e19f5 +Author: Yannick Welsch +Date: Mon Aug 6 08:34:38 2018 +0200 + + Handle AlreadyClosedException when bumping primary term + + If the shard is already closed while bumping the primary term, this can result in an + AlreadyClosedException to be thrown. As we use asyncBlockOperations, the exception + will be thrown on a thread from the generic thread pool and end up in the uncaught + exception handler, failing our tests. + + Relates to #32442 + +commit 66edba20122efdef2347b4d4db486c816f722d0a +Author: Ioannis Kakavas +Date: Mon Aug 6 07:42:26 2018 +0300 + + [TEST] Allow to run in FIPS JVM (#32607) + + * Change SecurityNioHttpServerTransportTests to use PEM key and + certificate files instead of a JKS keystore so that this tests + can also run in a FIPS 140 JVM + * Do not attempt to run cases with ssl.verification_mode NONE in + SessionFactoryTests so that the tests can run in a FIPS 140 JVM + +commit ceb1ae4d7b3c4dd45046a6cc2d364bd7bc69bd27 +Author: Ioannis Kakavas +Date: Mon Aug 6 03:00:45 2018 +0300 + + [Test] Add ckb to the list of unsupported languages (#32611) + +commit 1a39f1d6c5c8ea619119ec642e847c16aff9bcf8 +Author: Jason Tedor +Date: Sun Aug 5 08:53:08 2018 -0400 + + Fix CCR stats assertions + + This commit addresses a race that can happen in the basic CCR stats REST + tests. Namely, peek reads can fire before the REST test client fires the + stats request. This means that we have to weaken our assertions about + the expected stats response. + +commit 6fa7016bbfdba771e98fad92f389413276946dff +Author: Armin Braun +Date: Sat Aug 4 10:37:07 2018 +0200 + + SCRIPTING: Move Aggregation Scripts to their own context (#32068) + + * SCRIPTING: Move Aggregation Scripts to their own context + +commit 6ca24e13af3ac37a6803a4aeb533750270aecb11 +Author: Jack Conradson +Date: Fri Aug 3 15:22:30 2018 -0700 + + Painless: Use LocalMethod Map For Lookup at Runtime (#32599) + + This modifies Def to use a Map to look up user-defined methods at runtime + instead of writing constant methodhandles to do the reverse lookup. This creates a consistency + between how LocalMethods are looked up at compile-time and run-time. This consistency will allow + this code to be more maintainable moving forward. This will also allow FunctionReference to be + cleaned up in a follow up PR. + +commit 1e4751ec470ba5dc22257cf50a82a4b2bea63fce +Author: Lee Hinman +Date: Fri Aug 3 15:25:48 2018 -0600 + + [TEST] Enhance failure message when bulk updates have failures + +commit 457661e0dabee79cb5e0b4cc4842609e085f03da +Author: Tal Levy +Date: Fri Aug 3 14:00:27 2018 -0700 + + fix smoke test with security reference to renamed ilm + +commit b99aa81fe46a7d68a3e909db6ef67e2dc4d8144f +Author: David Roberts +Date: Fri Aug 3 20:48:38 2018 +0100 + + [ML] Add ML result classes to protocol library (#32587) + + This commit adds the ML results classes to the X-Pack protocol + library used by the high level REST client. + + (Other commits will add the config classes and stats classes.) + + These classes: + + - Are publically immutable + - Are privately mutable - this is perhaps not as nice as the + config classes, but to do otherwise would require adding + builders and the corresponding server-side classes that the + old transport client used don't have builders + - Have little/no validation of field values beyond null checks + - Are convertible to and from X-Content, but NOT wire transportable + - Have lenient parsers to maximize compatibility across versions + - Have the same class names and getter names as the corresponding + classes in X-Pack core to ease migration for transport client + users + - Don't reproduce all the methods that do calculations or + transformations that the the corresponding classes in X-Pack core + have + +commit 47a3e980b22cc02ffdbfa49fe9eafc5134efb658 +Merge: 78e94e1b000 992ec4be5de +Author: Lee Hinman +Date: Fri Aug 3 13:18:34 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit ada80d7fc857433790d5cdafd062ad9967a2afeb +Author: Igor Motov +Date: Fri Aug 3 11:59:51 2018 -0700 + + Suppress LicensingDocumentationIT.testPutLicense in release builds (#32613) + + The testPutLicense test tries to put a license generated using + snapshot keys into release cluster. This commit suppresses the + test during the release builds. + + Closes #32580 + +commit 992ec4be5dec3bfde7fcef5c1e1e81dad469ca4a +Author: Zachary Tong +Date: Fri Aug 3 14:09:01 2018 -0400 + + [Rollup] Update wire version check after backport + + Bumping down the version to 6.4 since the backport is complete. Also + adds some missing version checks to the bwc tests to make sure it + only runs on the correct versions + +commit 78e94e1b000031760a7f9921148f93e3d5adad2b +Merge: aadadb547ef e3cc33756e7 +Author: Tal Levy +Date: Fri Aug 3 11:06:29 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit e3cc33756e702a3a294c62af8aba9bb6597e92e4 +Author: David Turner +Date: Fri Aug 3 17:57:30 2018 +0100 + + Suppress Wildfly test in FIPS JVMs (#32543) + + WildflyIT fails on FIPS-enabled JVMs. This change mutes this test suite on such + JVMs. Relates #32534. + +commit fc9fb64ad5f9b16f9d3f8b88c5dbdc428ab5d1cd +Author: Zachary Tong +Date: Fri Aug 3 11:13:25 2018 -0400 + + [Rollup] Improve ID scheme for rollup documents (#32558) + + Previously, we were using a simple CRC32 for the IDs of rollup documents. + This is a very poor choice however, since 32bit IDs leads to collisions + between documents very quickly. + + This commit moves Rollups over to a 128bit ID. The ID is a concatenation + of all the keys in the document (similar to the rolling CRC before), + hashed with 128bit Murmur3, then base64 encoded. Finally, the job + ID and a delimiter (`$`) are prepended to the ID. + + This gurantees that there are 128bits per-job. 128bits should + essentially remove all chances of collisions, and the prepended + job ID means that _if_ there is a collision, it stays "within" + the job. + + BWC notes: + + We can only upgrade the ID scheme after we know there has been a good + checkpoint during indexing. We don't rely on a STARTED/STOPPED + status since we can't guarantee that resulted from a real checkpoint, + or other state. So we only upgrade the ID after we have reached + a checkpoint state during an active index run, and only after the + checkpoint has been confirmed. + + Once a job has been upgraded and checkpointed, the version increments + and the new ID is used in the future. All new jobs use the + new ID from the start + +commit aadadb547ef2bd6c8b57267754dc63458fa6a2cc +Author: Colin Goodheart-Smithe +Date: Fri Aug 3 15:31:26 2018 +0100 + + Fixes X-Pack loaded YAML test + +commit 3d4c84f7cac606ffc72b5df1d06881e5168c1b68 +Author: Jake Landis +Date: Fri Aug 3 07:21:05 2018 -0700 + + ingest: doc: move Dot Expander Processor doc to correct position (#31743) + + No changes to the content. + +commit 32c2759bb9163461879db19c18b35c2d48f5c78e +Author: Jason Tedor +Date: Fri Aug 3 09:55:04 2018 -0400 + + Remove extra blank line in CcrStatsAction.java + + This commit removes an extra blank line that was accidentally committed + to CcrStatsAction.java. + +commit d640c9ddf94f8aee51cbff1404128ac172fbda42 +Author: Jason Tedor +Date: Fri Aug 3 09:09:45 2018 -0400 + + Introduce CCR stats endpoint (#32350) + + This commit introduces the CCR stats endpoint which provides shard-level + stats on the status of CCR follower tasks. + +commit bc274b2ff2d01635bb6c30f20752e545763f49df +Author: David Roberts +Date: Fri Aug 3 13:21:08 2018 +0100 + + [ML] Add some ML config classes to protocol library (#32502) + + This commit adds four ML config classes to the X-Pack protocol + library used by the high level REST client. + + (Other commits will add the remaining config classes, plus results + and stats classes.) + + These classes: + + - Are immutable + - Have little/no validation of field values beyond null checks + - Are convertible to and from X-Content, but NOT wire transportable + - Have lenient parsers to maximize compatibility across versions + - Have the same class names, member names and getter/setter names + as the corresponding classes in X-Pack core to ease migration + for transport client users + - Don't reproduce all the methods that do calculations or + transformations that the the corresponding classes in X-Pack core + have + +commit 9a81b9dbd8d4930801985801d87d5581a16621df +Author: Colin Goodheart-Smithe +Date: Fri Aug 3 13:19:30 2018 +0100 + + adds index_lifecycle to ignored API conventions + +commit 1ee6393117bc0bbf448f24e31abe97ec1ea9f9dc +Author: Ioannis Kakavas +Date: Fri Aug 3 14:44:40 2018 +0300 + + [TEST]Split transport verification mode none tests (#32488) + + This commit splits SecurityNetty4TransportTests in two methods + one handling verification mode certificate and full and one + handling verification mode none. This is done so that the second + method can be muted in a FIPS 140 JVM where verification mode none + cannot be used. + +commit 20915a9bafddf627fc65c182780379e2e0790b9a +Merge: 0a6a48b91bd d05f39de8be +Author: Colin Goodheart-Smithe +Date: Fri Aug 3 12:25:38 2018 +0100 + + Merge branch 'master' into index-lifecycle + + client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClien + t.java + /Users/colings86/dev/work/git/elasticsearch/.git/worktrees/elasticsearch + -ilm/MERGE_HEAD + + client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLe + velClient.java + client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClien + t.java + client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifec + ycleIT.java + client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLe + velClientTests.java + client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT. + java + client/rest-high-level/src/test/java/org/elasticsearch/client/documentat + ion/LicensingDocumentationIT.java + client/rest-high-level/src/test/java/org/elasticsearch/client/documentat + ion/WatcherDocumentationIT.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler. + java + modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionR + ef.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessE + xplainError.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptCla + ssInfo.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Enh + ancedPainlessLexer.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/Pa + inlessLookup.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExp + licit.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFun + ctionRef.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EIns + tanceof.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELam + bda.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELis + tInit.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMap + Init.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENew + Array.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENew + Obj.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ESta + tic.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCal + lInvoke.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PFie + ld.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSub + ListShortcut.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSub + MapShortcut.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCat + ch.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDec + laration.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEac + h.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFun + ction.java + modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSub + EachIterable.java + modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadT + ests.java + modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessD + ocGenerator.java + modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTest + s.java + modules/lang-painless/src/test/java/org/elasticsearch/painless/node/Node + ToStringTests.java + rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_bas + ic.yml + server/src/main/java/org/elasticsearch/action/admin/cluster/stats/Cluste + rStatsResponse.java + server/src/main/java/org/elasticsearch/action/admin/cluster/stats/Transp + ortClusterStatsAction.java + server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAct + ion.java + server/src/main/java/org/elasticsearch/action/support/replication/Transp + ortReplicationAction.java + server/src/main/java/org/elasticsearch/index/engine/Engine.java + server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java + server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.ja + va + server/src/main/java/org/elasticsearch/index/shard/IndexShard.java + server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPe + rmits.java + server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer. + java + server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java + server/src/main/java/org/elasticsearch/index/translog/Translog.java + server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHa + ndler.java + server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.j + ava + server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkAct + ionTests.java + server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTe + sts.java + server/src/test/java/org/elasticsearch/action/support/replication/Transp + ortReplicationActionTests.java + server/src/test/java/org/elasticsearch/action/support/replication/Transp + ortWriteActionTests.java + server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardS + tateIT.java + server/src/test/java/org/elasticsearch/index/replication/IndexLevelRepli + cationTests.java + server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringR + eplicationTests.java + server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTes + ts.java + server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPe + rmitsTests.java + server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java + server/src/test/java/org/elasticsearch/index/shard/IndexingOperationList + enerTests.java + server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerT + ests.java + server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java + server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHa + ndlerTests.java + server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.ja + va + server/src/test/java/org/elasticsearch/search/profile/query/QueryProfile + rIT.java + test/framework/src/main/java/org/elasticsearch/index/replication/ESIndex + LevelReplicationTestCase.java + test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTes + tCase.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job + /MetricConfig.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job + /RollupJobConfig.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/S + ecurityContext.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/U + serSettings.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + ction/user/AuthenticateResponse.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + ction/user/ChangePasswordRequestBuilder.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + ction/user/GetUsersResponse.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + ction/user/PutUserRequestBuilder.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthc/Authentication.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthc/AuthenticationResult.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthc/Realm.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthz/RoleDescriptor.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthz/accesscontrol/SecurityIndexSearcherWrapper.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/s + upport/MetadataUtils.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/AnonymousUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/BeatsSystemUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/ElasticUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/InternalUserSerializationHelper.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/KibanaUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/LogstashSystemUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/SystemUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/XPackSecurityUser.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/XPackUser.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/Con + figTestHelpers.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job + /MetricsConfigSerializingTests.java -> + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job + /MetricConfigSerializingTests.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/a + uthz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java + x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitorin + g/collector/cluster/ClusterStatsMonitoringDocTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/Rollup + JobIdentifierUtilTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/Rollup + RequestTranslationTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action + /SearchActionTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config + /ConfigTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/In + dexerUtilsTests.java + x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/Ro + llupIndexerIndexingTests.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ac + tion/saml/TransportSamlLogoutAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ac + tion/user/TransportAuthenticateAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ac + tion/user/TransportGetUsersAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ac + tion/user/TransportHasPrivilegesAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + dit/AuditTrail.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + dit/AuditTrailService.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + dit/index/IndexAuditTrail.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + dit/logfile/LoggingAuditTrail.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/AuthenticationService.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/esnative/NativeRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/esnative/NativeUsersStore.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/esnative/ReservedRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/esnative/UserAndPassword.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/file/FileRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/file/FileUserPasswdStore.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/kerberos/KerberosRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/ldap/LdapRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/pki/PkiRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/saml/SamlRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thc/support/CachingUsernamePasswordRealm.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thz/AuthorizationService.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thz/AuthorizedIndices.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/in + gest/SetSecurityUserProcessor.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/re + st/action/RestAuthenticateAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/re + st/action/user/RestChangePasswordAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/re + st/action/user/RestGetUsersAction.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tr + ansport/ServerTransportFilter.java + x-pack/plugin/security/src/test/java/org/elasticsearch/integration/Clear + RealmsCacheTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/Se + curityContextTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/filter/SecurityActionFilterTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/interceptor/IndicesAliasesRequestInterceptorTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/interceptor/ResizeRequestInterceptorTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/saml/TransportSamlInvalidateSessionActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/saml/TransportSamlLogoutActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportAuthenticateActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportChangePasswordActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportDeleteUserActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportGetUsersActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportHasPrivilegesActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportPutUserActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ac + tion/user/TransportSetEnabledActionTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + dit/AuditTrailServiceTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + dit/index/IndexAuditTrailMutedTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + dit/index/IndexAuditTrailTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + dit/logfile/LoggingAuditTrailFilterTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + dit/logfile/LoggingAuditTrailTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/AuthenticationServiceTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/RealmsTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/TokenServiceTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/UserTokenTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/esnative/NativeRealmIntegTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/esnative/NativeUsersStoreTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/esnative/ReservedRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/file/FileRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/file/FileUserPasswdStoreTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/kerberos/KerberosRealmAuthenticateFailedTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/kerberos/KerberosRealmCacheTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/kerberos/KerberosRealmTestCase.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/kerberos/KerberosRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/ldap/ActiveDirectoryRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/ldap/CancellableLdapRunnableTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/ldap/LdapRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/pki/PkiRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/support/CachingUsernamePasswordRealmTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thc/support/mapper/NativeRoleMappingStoreTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thz/AuthorizationServiceTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thz/AuthorizationUtilsTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thz/AuthorizedIndicesTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thz/IndicesAndAliasesResolverTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/au + thz/SecuritySearchOperationListenerTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/in + gest/SetSecurityUserProcessorTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/tr + ansport/SecurityServerTransportInterceptorTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/tr + ansport/ServerTransportFilterTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/us + er/AnonymousUserTests.java + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/us + er/UserTests.java -> + x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/us + er/UserSerializationTests.java + x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/f + unction/scalar/string/StringFunctionUtils.java + x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/exec + ution/ExecutionServiceTests.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/DefaultDetectorDescription.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/DetectionRule.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/Detector.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/DetectorFunction.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/FilterRef.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/MlFilter.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/Operator.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/RuleAction.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/RuleCondition.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/RuleScope.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/packag + e-info.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/u + ser/User.java -> + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/ + User.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/DetectionRuleTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/DetectorTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/FilterRefTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/MlFilterTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/RuleConditionTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/co + nfig/RuleScopeTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/security/ + UserTests.java + x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/Watch + erRestartIT.java + x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cl + uster/60_watcher.yml + x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_clus + ter/60_watcher.yml + x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded + _cluster/60_watcher.yml + x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch + /example/realm/CustomRealm.java + x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch + /example/realm/CustomRealmTests.java + x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/s + ecurity/MigrateToolIT.java + +commit 018e77cac69793adac1502937d78f88c06151e93 +Author: Alexander Reelsen +Date: Fri Aug 3 13:21:14 2018 +0200 + + Core: Move helper date formatters over to java time (#32504) + + Some classes use internal date formatters, which now can be moved over + to java time using the DateFormatters class. + + The same applies for a few test cases. + +commit 21f660d801a1f08981a1806d747b8665e63ead7e +Author: Tanguy Leroux +Date: Fri Aug 3 13:11:00 2018 +0200 + + [Rollup] Remove builders from DateHistogramGroupConfig (#32555) + + Same motivation as #32507 but for the DateHistogramGroupConfig + configuration object. This pull request also changes the format of the + time zone from a Joda's DateTimeZone to a simple String. + + It should help to port the API to the high level rest client and allows + clients to not be forced to use the Joda Time library. Serialization is + impacted but does not need a backward compatibility layer as + DateTimeZone are serialized as String anyway. XContent also expects + a String for timezone, so I found it easier to move everything to String. + + Related to #29827 + +commit d05f39de8be1286b6360b327a3e9d1645fa2cd36 +Author: Colin Goodheart-Smithe +Date: Fri Aug 3 11:47:02 2018 +0100 + + [TEST} unmutes SearchAsyncActionTests and adds debugging info + + This unmutes the testFanOutAndCollect()` method and add a check to make + sure we aren't accidentally running something twice causing a search + phase to still be running after we have counted down the latch + + Relates to #29242 + +commit eb17128b9c43ba0dfde374764f4c1e8b92a44a0e +Author: David Roberts +Date: Fri Aug 3 10:39:29 2018 +0100 + + [ML] Add Detector config classes to protocol library (#32495) + + This commit adds the Detector class and its dependencies to the + X-Pack protocol library used by the high level REST client. + + (Future commits will add the remaining config classes, plus results + and stats classes.) + + These classes: + + - Are immutable, with builders, but the builders do no validation + beyond null checks + - Are convertible to and from X-Content, but NOT wire transportable + - Have lenient parsers to maximize compatibility across versions + - Have the same class names, member names and getter/setter names + as the corresponding classes in X-Pack core to ease migration + for transport client users + - Don't reproduce all the methods that do calculations or + transformations that the the corresponding classes in X-Pack core + have + +commit 937dcfd716de906e007152aa2e0d9dee2b779672 +Author: Tanguy Leroux +Date: Fri Aug 3 10:01:20 2018 +0200 + + [Rollup] Remove builders from MetricConfig (#32536) + + Related to #29827 + +commit f809d6fff415626b5b9d4f393f9caa638be7f000 +Author: Alexander Reelsen +Date: Fri Aug 3 09:41:29 2018 +0200 + + Tests: Add rolling upgrade tests for watcher (#32428) + + These tests ensure, that the basic watch APIs are tested in the rolling + upgrade tests. After initially adding a watch, the tests try to get, + execute, deactivate and activate a watch. Watcher stats are tested as + well, and an own java based test has been added for restarting, as that + requires waiting for a state change. Watcher history is also checked. + + Closes #31216 + +commit 0d60e8a029a8e38cbf69c2d923f0b97dc144c8af +Author: Yannick Welsch +Date: Fri Aug 3 09:33:08 2018 +0200 + + Fix race between replica reset and primary promotion (#32442) + + We've recently seen a number of test failures that tripped an assertion in IndexShard (see issues + linked below), leading to the discovery of a race between resetting a replica when it learns about a + higher term and when the same replica is promoted to primary. This commit fixes the race by + distinguishing between a cluster state primary term (called pendingPrimaryTerm) and a shard-level + operation term. The former is set during the cluster state update or when a replica learns about a + new primary. The latter is only incremented under the operation block, which can happen in a + delayed fashion. It also solves the issue where a replica that's still adjusting to the new term + receives a cluster state update that promotes it to primary, which can happen in the situation of + multiple nodes being shut down in short succession. In that case, the cluster state update thread + would call `asyncBlockOperations` in `updateShardState`, which in turn would throw an exception + as blocking permits is not allowed while an ongoing block is in place, subsequently failing the shard. + This commit therefore extends the IndexShardOperationPermits to allow it to queue multiple blocks + (which will all take precedence over operations acquiring permits). Finally, it also moves the primary + activation of the replication tracker under the operation block, so that the actual transition to + primary only happens under the operation block. + + Relates to #32431, #32304 and #32118 + +commit 6eeb628d6d0da2daef3ae93091f974e8f685d39a +Merge: 2387616c80b 9dcf3f5aee6 +Author: Nhat Nguyen +Date: Thu Aug 2 23:14:37 2018 -0400 + + Merge branch 'master' into ccr + + * master: + HLRC: Move commercial clients from XPackClient (#32596) + Add cluster UUID to Cluster Stats API response (#32206) + Security: move User to protocol project (#32367) + [TEST] Test for shard failures, add debug to testProfileMatchesRegular + Minor fix for javadoc (applicable for java 11). (#32573) + Painless: Move Some Lookup Logic to PainlessLookup (#32565) + TEST: Avoid merges in testSeqNoAndCheckpoints + [Rollup] Remove builders from HistoGroupConfig (#32533) + Mutes failing SQL string function tests due to #32589 + fixed elements in array of produced terms (#32519) + INGEST: Enable default pipelines (#32286) + Remove cluster state initial customs (#32501) + Mutes LicensingDocumentationIT due to #32580 + [ML] Remove multiple_bucket_spans (#32496) + [ML] Rename JobProvider to JobResultsProvider (#32551) + Correct minor typo in explain.asciidoc for HLRC + Build: Add elastic maven to repos used by BuildPlugin (#32549) + Clarify the error message when a pipeline agg is used in the 'order' parameter. (#32522) + Revert "[test] turn on host io cache for opensuse (#32053)" + Enable packaging tests on suse boxes + [ML] Improve error when no available field exists for rule scope (#32550) + [ML] Improve error for functions with limited rule condition support (#32548) + Painless: Clean Up PainlessField (#32525) + Add @AwaitsFix for #32554 + Remove broken @link in Javadoc + Scripting: Conditionally use java time api in scripting (#31441) + [ML] Fix thread leak when waiting for job flush (#32196) (#32541) + Add AwaitsFix to failing test - see #32546 + Core: Minor size reduction for AbstractComponent (#32509) + SQL: Added support for string manipulating functions with more than one parameter (#32356) + [DOCS] Reloadable Secure Settings (#31713) + Watcher: Reenable HttpSecretsIntegrationTests#testWebhookAction test (#32456) + [Rollup] Remove builders from TermsGroupConfig (#32507) + Use hostname instead of IP with SPNEGO test (#32514) + Switch x-pack rolling restart to new style Requests (#32339) + NETWORKING: Fix Netty Leaks by upgrading to 4.1.28 (#32511) + [DOCS] Small fixes in rule configuration page (#32516) + Painless: Clean up PainlessMethod (#32476) + Build: Remove shadowing from benchmarks (#32475) + Docs: Add all JDKs to CONTRIBUTING.md + Add licensing enforcement for FIPS mode (#32437) + SQL: Add test for handling of partial results (#32474) + Mute testFilterCacheStats + [ML][DOCS] Fix typo applied_to => applies_to + Scripting: Fix painless compiler loader to know about context classes (#32385) + +commit 9dcf3f5aee68d84fa35bd62b866f001afce33890 +Author: Michael Basnight +Date: Thu Aug 2 19:38:49 2018 -0500 + + HLRC: Move commercial clients from XPackClient (#32596) + + The commercial clients were improperly placed into XPackClient, which is + a wrapper for the miscellaneous usage and info APIs. This commit moves + them into the HLRC. + +commit 2387616c80b593064beeae5eada95e5dddb1082b +Author: Jason Tedor +Date: Thu Aug 2 20:21:43 2018 -0400 + + Remove _xpack from CCR APIs (#32563) + + For a new feature like CCR we will go without this extra layer of + indirection. This commit replaces all /_xpack/ccr/_(\S+) endpoints by + /_ccr/$1 endpoints. + +commit 0a83968650d93144cabec755baf5ec1d93754aba +Author: Shaunak Kashyap +Date: Thu Aug 2 17:14:19 2018 -0700 + + Add cluster UUID to Cluster Stats API response (#32206) + + * Make cluster stats response contain cluster UUID + + * Updating constructor usage in Monitoring tests + + * Adding cluster_uuid field to Cluster Stats API reference doc + + * Adding rest api spec test for expecting cluster_uuid in cluster stats response + + * Adding missing newline + + * Indenting do section properly + + * Missed a spot! + + * Fixing the test cluster ID + +commit 0a6a48b91bd8274c101bf9fb97de88a05007c95c +Author: Lee Hinman +Date: Thu Aug 2 15:27:02 2018 -0600 + + [TEST] Fix HLRC using pre-rename endpoint for setting lifecycle policy + +commit eb3accb72118f2c85eacd6f598754fcd2e9a1f07 +Author: Jay Modi +Date: Thu Aug 2 15:21:34 2018 -0600 + + Security: move User to protocol project (#32367) + + The User class has been moved to the protocol project for upcoming work + to add more security APIs to the high level rest client. As part of + this change, the toString method no longer uses a custom output method + from MetadataUtils and instead just relies on Java's toString + implementation. + +commit 080b9f58ea757a194053d09f49e3d0c9898cbe18 +Author: Zachary Tong +Date: Thu Aug 2 17:16:34 2018 -0400 + + [TEST] Test for shard failures, add debug to testProfileMatchesRegular + + Unmuting the test and adding some more debug output. Was not able to + reproduce the prior failure, but it seems possible that the + failure (mismatched counts) could be caused by partial search results + during the test. + + The assertions check for shard failures first, because if one of the + two searches is partial the rest of the test will fail. + + Next, instead of just checking respective hit counts, we emit the + difference in hits to help identify what went wrong. + + Closes #32492 + +commit 9e1e38ff51e3ace950f6e1c3450554bbf20d6484 +Author: Andrei Stefan +Date: Thu Aug 2 23:46:08 2018 +0300 + + Minor fix for javadoc (applicable for java 11). (#32573) + +commit 9f96073e645f10237a5845804963e4c949b29a7c +Author: Nhat Nguyen +Date: Thu Aug 2 15:44:40 2018 -0400 + + TEST: Avoid merges in testRecoveryWithOutOfOrderDelete + + Since LUCENE-8263, testRecoveryWithOutOfOrderDelete may trigger merges + because of the deletes. In the test, we try to retain index#0 but + reclaim delete#1. However, if a merge is triggered, we will remove both + index#0 and delete#1. + + This commit disables merges in this test. Another option is to index + more documents in the segment_2 to reduce the deletion ratio. + +commit b938960602a559cf9e1133779cc9d7e2b0ec401a +Author: Jack Conradson +Date: Thu Aug 2 12:33:25 2018 -0700 + + Painless: Move Some Lookup Logic to PainlessLookup (#32565) + + Renames existing methods in PainlessLookup. Adds lookupPainlessClass, + lookupPainlessMethod, and lookupPainlessField to PainlessLookup. This consolidates + the logic necessary to look these things up into a single place and begins the clean up of + some of the nodes that were looking each of these things up individually. This also has + the added benefit of improved consistency in error messaging. + +commit 6986c9f700b81f4f15d8404f03d026a8672fac3d +Merge: f7162588dff 2c35db8043d +Author: Lee Hinman +Date: Thu Aug 2 12:29:28 2018 -0600 + + Merge remote-tracking branch 'origin/master' into index-lifecycle + +commit 2c35db8043d7ebe314519031b0ad2f6333d72535 +Author: Nhat Nguyen +Date: Thu Aug 2 13:45:06 2018 -0400 + + TEST: Avoid merges in testSeqNoAndCheckpoints + + Since LUCENE-8263, testSeqNoAndCheckpoints might trigger merges because + of the updates and deletes in the test. Our merge scheduler will trigger + a flush if there is no pending merge. Those extra flushes will change + the last committed segmentInfos in the engine and fail the test. + + This commit uses LogMergePolicy for the engine in the test to avoid + merges. + + Closes #32430 + +commit f7162588dfff4073d848d7179253495fedef926f +Merge: 5de236e1e7d 08e4f4be425 +Author: Tal Levy +Date: Thu Aug 2 10:22:46 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 5de236e1e7d4e70ed3a498109dc54b92631fe998 +Author: Jason Tedor +Date: Thu Aug 2 13:05:11 2018 -0400 + + Rename ILM, ILM endpoints and drop _xpack (#32564) + + This commit does the following: + - renames index-lifecycle plugin to ilm + - modifies the endpoints to ilm instead of index_lifecycle + - drops _xpack from the endpoints + - drops a few duplicate endpoints + +commit 08e4f4be42566c24370c7bedf8ed8d3611f29939 +Author: Tanguy Leroux +Date: Thu Aug 2 17:55:00 2018 +0200 + + [Rollup] Remove builders from HistoGroupConfig (#32533) + + Related to #29827 + +commit 1bb7b53f267c715235faac04f64e82132ef81843 +Author: Lee Hinman +Date: Thu Aug 2 09:51:08 2018 -0600 + + Make index.lifecycle.name setting internal (#32518) + + This commit makes the `index.lifecycle.name` setting internal an index, this + means that the policy can only be set on the index creation, or with the + specialized `RestSetIndexLifecyclePolicy` action. + + Relates to #29823 + +commit 4cdbb425425e9a8f3280bcf72430ab03d4bce461 +Author: Colin Goodheart-Smithe +Date: Thu Aug 2 16:15:10 2018 +0100 + + Mutes failing SQL string function tests due to #32589 + +commit a39f6f09f4559707ae7302c5462c60ce78f4e7c3 +Author: Matthias Sieber +Date: Thu Aug 2 08:12:15 2018 -0700 + + fixed elements in array of produced terms (#32519) + +commit be31cc642b4a677019e1736141dd11e7bad02ff9 +Author: Armin Braun +Date: Thu Aug 2 17:11:12 2018 +0200 + + INGEST: Enable default pipelines (#32286) + + * INGEST: Enable default pipelines + + * Add `default_pipeline` index setting + * `_none` is interpreted as no pipeline + * closes #21101 + +commit db6e8c736d92582fe56024993450ce0a987b498d +Author: Yannick Welsch +Date: Thu Aug 2 15:49:59 2018 +0200 + + Remove cluster state initial customs (#32501) + + This infrastructure was introduced in #26144 and made obsolete in #30743 + +commit 097c4287f3ca61e171970fb98931e6b46fe1dbd2 +Author: Colin Goodheart-Smithe +Date: Thu Aug 2 11:58:15 2018 +0100 + + Mutes LicensingDocumentationIT due to #32580 + +commit f30bb0ebf8b676880276801737504fcb95ac1534 +Author: Dimitris Athanasiou +Date: Thu Aug 2 11:25:56 2018 +0100 + + [ML] Remove multiple_bucket_spans (#32496) + + This commit removes the never released multiple_bucket_spans + configuration parameter. This is now replaced with the new + multibucket feature that requires no configuration. + +commit 15679315e3e75d58e6834a8cfd65bb258874fdfc +Author: David Kyle +Date: Thu Aug 2 09:53:47 2018 +0100 + + [ML] Rename JobProvider to JobResultsProvider (#32551) + +commit 43430fe9b860641a306a448539984c88309505da +Author: Mayya Sharipova +Date: Wed Aug 1 17:58:14 2018 -0400 + + Correct minor typo in explain.asciidoc for HLRC + +commit 781bb8efe527ceb865f314aa9f81f5016cdad2a3 +Author: Ryan Ernst +Date: Wed Aug 1 13:48:13 2018 -0700 + + Build: Add elastic maven to repos used by BuildPlugin (#32549) + + This commit adds the elastic repo, alongside maven central, to any + plugin using BuildPlugin. This is necessary now because the default + distribution, which the test framework uses by default, is now only + hosted on elastic maven. While inside the elasticsearch build this does + not matter, those that build external plugins with our build-tools could + have tests fail to find the distribution dependency. + +commit 5efc2ec9f70da95ea6d7a7ba9ce65fa93382da04 +Author: Julie Tibshirani +Date: Wed Aug 1 12:02:07 2018 -0700 + + Clarify the error message when a pipeline agg is used in the 'order' parameter. (#32522) + +commit 16fe22047e4ce243e9858da568b4e7a8eb73ef35 +Author: Andy Bristol +Date: Wed Aug 1 11:29:13 2018 -0700 + + Revert "[test] turn on host io cache for opensuse (#32053)" + + This reverts commit 69c85331b0f574323994dc1ccb4f70ab0e122a4c. + +commit 418540e630280d7e4f899ebff2f5202295ac0de0 +Author: Andy Bristol +Date: Wed Aug 1 11:25:36 2018 -0700 + + Enable packaging tests on suse boxes + + This reverts commit 0f7d2044c321d49c2e1ed71273d4f39fc7cfeb9a. + +commit 8bf83647f595f82fb33d2faebf6f8f1b42c09851 +Author: Dimitris Athanasiou +Date: Wed Aug 1 18:58:20 2018 +0100 + + [ML] Improve error when no available field exists for rule scope (#32550) + + Closes #32542 + +commit c5140170f7978a3072a635ce0e89be006807726c +Author: Dimitris Athanasiou +Date: Wed Aug 1 18:15:46 2018 +0100 + + [ML] Improve error for functions with limited rule condition support (#32548) + + Closes #32545 + +commit ea1e26ca2716210395390170c982ba05d8425a37 +Merge: 304304fe41f 29859201349 +Author: Tal Levy +Date: Wed Aug 1 10:15:18 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 298592013494f311ac1f1358bcd816322feb2122 +Author: Jack Conradson +Date: Wed Aug 1 09:28:18 2018 -0700 + + Painless: Clean Up PainlessField (#32525) + + Updates PainlessField variable names to current naming scheme and removes + extraneous variables. + +commit 2db420a81d5a4b531816ce03fa2b401c12431a32 +Author: David Turner +Date: Wed Aug 1 17:26:45 2018 +0100 + + Add @AwaitsFix for #32554 + +commit 621350c1b973d8c02afe62b6130231f46ae1affc +Author: David Turner +Date: Wed Aug 1 17:12:55 2018 +0100 + + Remove broken @link in Javadoc + +commit 478f6d6cf17c52b09af5f56663746a61190d6c0d +Author: Ryan Ernst +Date: Wed Aug 1 08:58:49 2018 -0700 + + Scripting: Conditionally use java time api in scripting (#31441) + + This commit adds a boolean system property, `es.scripting.use_java_time`, + which controls the concrete return type used by doc values within + scripts. The return type of accessing doc values for a date field is + changed to Object, essentially duck typing the type to allow + co-existence during the transition from joda time to java time. + +commit 9fb790dcc362a70ec0896313658bac1fcead2334 +Author: Benjamin Trent +Date: Wed Aug 1 10:38:04 2018 -0500 + + [ML] Fix thread leak when waiting for job flush (#32196) (#32541) + +commit 7a3eb4b3e87d2dcb1c3b05f902bb69d6d62d12e9 +Author: David Turner +Date: Wed Aug 1 14:28:00 2018 +0100 + + Add AwaitsFix to failing test - see #32546 + +commit e7ead178931091c00059c86019d68f164daf1783 +Author: Nik Everett +Date: Wed Aug 1 09:17:48 2018 -0400 + + Core: Minor size reduction for AbstractComponent (#32509) + + This removes a constructor from `AbstractComponent` and + `AbstractLifecycleComponent` that we weren't using and it switches the + logger creation away from one of the `Settings` flavored methods which + are no longer needed. + +commit 4c388539a0f69f1cdf74424769fb36ea80631170 +Author: Andrei Stefan +Date: Wed Aug 1 12:29:06 2018 +0300 + + SQL: Added support for string manipulating functions with more than one parameter (#32356) + + Added support for string manipulating functions with more than one parameter: + CONCAT, LEFT, RIGHT, REPEAT, POSITION, LOCATE, REPLACE, SUBSTRING, INSERT + +commit d80b639c188085e738bea1254b89b8f589d02ce4 +Merge: d2092100199 2d87287c0d7 +Author: Yannick Welsch +Date: Wed Aug 1 11:07:44 2018 +0200 + + Merge remote-tracking branch 'elastic/master' into zen2 + +commit 2d87287c0d768256e34a94ddbb1dcd2219eab321 +Author: Albert Zaharovits +Date: Wed Aug 1 12:07:23 2018 +0300 + + [DOCS] Reloadable Secure Settings (#31713) + + Docs on reloadable secure settings for plugins #29135 . + +commit 7d7cddc6ed5646ea277c0591f78fb16ebfb9f4c2 +Author: Alexander Reelsen +Date: Wed Aug 1 10:46:02 2018 +0200 + + Watcher: Reenable HttpSecretsIntegrationTests#testWebhookAction test (#32456) + + The error message mentioned in #30094 does not link to to a cause by the + test itself, as there are still inflight requests according to the + circuit breaker. + + I ran this test class 100k times on bare metal and could not reproduce + it. I will reenable the test for now. + + Closes #30094 + +commit 82fe67b2250567dbe5a7cc8c9021af0c5fa4da38 +Author: Tanguy Leroux +Date: Wed Aug 1 09:43:32 2018 +0200 + + [Rollup] Remove builders from TermsGroupConfig (#32507) + + While working on adding the Create Rollup Job API to the + high level REST client (#29827), I noticed that the configuration + objects like TermsGroupConfig rely on the Builder pattern in + order to create or parse instances. These builders are doing + some validation but the same validation could be done within + the constructor itself or on the server side when appropriate. + + This commit removes the builder for TermsGroupConfig, + removes some other methods that I consider not really usefull + once the TermsGroupConfig object will be exposed in the + high level REST client. It also simplifies the parsing logic. + + Related to #29827 + +commit 304304fe41fa855a58401f4209feed4374618455 +Author: Tal Levy +Date: Tue Jul 31 21:06:04 2018 -0700 + + re-enable ILM integration tests and fix policyRegistry update bug (#32108) + + This PR re-introduces our ILM integration tests with mock steps + that we can control in the tests. + + These tests uncovered a bug where the policy-steps-registry was + not being updated on newly elected masters when there were no + cluster-state changes to ILM metadata. The fix layed out cleans up + the registry/runner when a node is un-elected as master. It re-assigns + the class variables so that the existing runner/registry instances that + may be running can continue to do so in other threads, potentially. + +commit f2f33f31497654dc0fd687dd480e1636c898b566 +Author: Jay Modi +Date: Tue Jul 31 20:57:33 2018 -0600 + + Use hostname instead of IP with SPNEGO test (#32514) + + This change updates KerberosAuthenticationIT to resolve the host used + to connect to the test cluster. This is needed because the host could + be an IP address but SPNEGO requires a hostname to work properly. This + is done by adding a hook in ESRestTestCase for building the HttpHost + from the host and port. + + Additionally, the project now specifies the IPv4 loopback address as + the http host. This is done because we need to be able to resolve the + address used for the HTTP transport before the node starts up, but the + http.ports file is not written until the node is started. + + Closes #32498 + +commit 99d9a0a40c53b628bcff4353ae7e80ccb934769b +Author: Nik Everett +Date: Tue Jul 31 21:04:33 2018 -0400 + + Switch x-pack rolling restart to new style Requests (#32339) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack:qa:rolling-upgrade*` projects to use + the new versions. + +commit 4b199dde8d245a2e0b81e0b556eb22eab637cecf +Author: Armin Braun +Date: Wed Aug 1 02:34:58 2018 +0200 + + NETWORKING: Fix Netty Leaks by upgrading to 4.1.28 (#32511) + + * Upgrade to `4.1.28` since the problem reported in #32487 is a bug in Netty itself (see https://github.com/netty/netty/issues/7337) + * Fixed other leaks in test code that now showed up due to fixes improvements in leak reporting in the newer version + * Needed to extend permissions for netty common package because it now sets a classloader at runtime after changes in https://github.com/netty/netty/commit/63bae0956aeb42e875ad303a2281989ad92acbf2 + * Adjusted forbidden APIs check accordingly + * Closes #32487 + +commit cc6d6cae7cee3620ff72fdca909515b0c1916553 +Author: Lisa Cawley +Date: Tue Jul 31 16:28:09 2018 -0700 + + [DOCS] Small fixes in rule configuration page (#32516) + +commit 09e38f2f5994b45acf7cd6b7f512bc50dffca4c9 +Author: Jack Conradson +Date: Tue Jul 31 16:25:53 2018 -0700 + + Painless: Clean up PainlessMethod (#32476) + + Renames and removes variables from PainlessMethod to follow the new naming + convention. Generates methodtypes at compile-time instead of using a method at run- + time. Moves write method to MethodWriter. + +commit 9a0f6b2c38bd37b2189aa20c7a83dcce52355429 +Author: Tal Levy +Date: Tue Jul 31 15:34:34 2018 -0700 + + remove mention of old maintenance mode setting + +commit 21eb9695afe422cbdbbc7af098c602cb74dbfe50 +Author: Nik Everett +Date: Tue Jul 31 17:31:13 2018 -0400 + + Build: Remove shadowing from benchmarks (#32475) + + Removes shadowing from the benchmarks. It isn't *strictly* needed. We do + have to rework the documentation on how to run the benchmark, but it + still seems to work if you run everything through gradle. + +commit b2c2c94741b991575a4bd3616aba4c3de3b33f10 +Author: aaronoah +Date: Mon Jul 30 19:42:18 2018 -0500 + + Docs: Add all JDKs to CONTRIBUTING.md + +commit 73cdfaf07be5c8afe00c5dd4e5c6741b152086d5 +Merge: 6e9f33843d1 07881885745 +Author: Tal Levy +Date: Tue Jul 31 11:50:33 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 0788188574583c6b3cabb27b374b0ea6f97f5ee7 +Author: Jay Modi +Date: Tue Jul 31 12:16:22 2018 -0600 + + Add licensing enforcement for FIPS mode (#32437) + + This commit adds licensing enforcement for FIPS mode through the use of + a bootstrap check, a node join validator, and a check in the license + service. The work done here is based on the current implementation of + the TLS enforcement with a production license. + + The bootstrap check is always enforced since we need to enforce the + licensing and this is the best option to do so at the present time. + +commit 5fd72028087a08eb38235adb91cca671b16bcae0 +Author: Igor Motov +Date: Tue Jul 31 11:06:00 2018 -0700 + + SQL: Add test for handling of partial results (#32474) + + Verifies that partial results are rejected by SQL requests. + + Closes #32284 + +commit 67d53e509361aa6795089a8c5fe01c771c006fba +Author: Nhat Nguyen +Date: Tue Jul 31 12:45:30 2018 -0400 + + Mute testFilterCacheStats + + Tracked at #32506 + +commit b88b3d81c87c27f7ed5e5b4690f38c96574760ce +Author: Dimitris Athanasiou +Date: Tue Jul 31 17:00:55 2018 +0100 + + [ML][DOCS] Fix typo applied_to => applies_to + +commit 2ed9782a678003c34ca536207463e2d8f860a74c +Author: Ryan Ernst +Date: Tue Jul 31 08:28:03 2018 -0700 + + Scripting: Fix painless compiler loader to know about context classes (#32385) + + This commit fixes the painless compiler classloader to know about the + classes from the script context. This fixes an issue when a custom + context is used from a plugin which caused a ClassNotFoundException for + the script class and its factory classes. + +commit 036cb3f86451517d90cd4b742e83a2115c11ba5a +Merge: 8cfbb64d6e7 22459576d75 +Author: Nhat Nguyen +Date: Tue Jul 31 10:56:10 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Logging: Make node name consistent in logger (#31588) + Mute SSLTrustRestrictionsTests on JDK 11 + Increase max chunk size to 256Mb for repo-azure (#32101) + Docs: Fix README upgrade mention (#32313) + Changed ReindexRequest to use Writeable.Reader (#32401) + Mute KerberosAuthenticationIT + Fix AutoIntervalDateHistogram.testReduce random failures (#32301) + fix no=>not typo (#32463) + Mute QueryProfilerIT#testProfileMatchesRegular() + HLRC: Add delete watch action (#32337) + High-level client: fix clusterAlias parsing in SearchHit (#32465) + Fix calculation of orientation of polygons (#27967) + [Kerberos] Add missing javadocs (#32469) + [Kerberos] Remove Kerberos bootstrap checks (#32451) + Make get all app privs requires "*" permission (#32460) + Switch security to new style Requests (#32290) + Switch security spi example to new style Requests (#32341) + Painless: Add PainlessConstructor (#32447) + update rollover to leverage write-alias semantics (#32216) + Update Fuzzy Query docs to clarify default behavior re max_expansions (#30819) + INGEST: Clean up Java8 Stream Usage (#32059) + Ensure KeyStoreWrapper decryption exceptions are handled (#32464) + +commit 22459576d757833d9dfbca59cb44b09fc8bd77b1 +Author: Nik Everett +Date: Tue Jul 31 10:54:24 2018 -0400 + + Logging: Make node name consistent in logger (#31588) + + First, some background: we have 15 different methods to get a logger in + Elasticsearch but they can be broken down into three broad categories + based on what information is provided when building the logger. + + Just a class like: + ``` + private static final Logger logger = ESLoggerFactory.getLogger(ActionModule.class); + ``` + or: + ``` + protected final Logger logger = Loggers.getLogger(getClass()); + ``` + + The class and settings: + ``` + this.logger = Loggers.getLogger(getClass(), settings); + ``` + + Or more information like: + ``` + Loggers.getLogger("index.store.deletes", settings, shardId) + ``` + + The goal of the "class and settings" variant is to attach the node name + to the logger. Because we don't always have the settings available, we + often use the "just a class" variant and get loggers without node names + attached. There isn't any real consistency here. Some loggers get the + node name because it is convenient and some do not. + + This change makes the node name available to all loggers all the time. + Almost. There are some caveats are testing that I'll get to. But in + *production* code the node name is node available to all loggers. This + means we can stop using the "class and settings" variants to fetch + loggers which was the real goal here, but a pleasant side effect is that + the ndoe name is now consitent on every log line and optional by editing + the logging pattern. This is all powered by setting the node name + statically on a logging formatter very early in initialization. + + Now to tests: tests can't set the node name statically because + subclasses of `ESIntegTestCase` run many nodes in the same jvm, even in + the same class loader. Also, lots of tests don't run with a real node so + they don't *have* a node name at all. To support multiple nodes in the + same JVM tests suss out the node name from the thread name which works + surprisingly well and easy to test in a nice way. For those threads + that are not part of an `ESIntegTestCase` node we stick whatever useful + information we can get form the thread name in the place of the node + name. This allows us to keep the logger format consistent. + +commit 5f302580f9dea18b389007b861fc06b013e94499 +Author: Daniel Mitterdorfer +Date: Tue Jul 31 16:17:28 2018 +0200 + + Mute SSLTrustRestrictionsTests on JDK 11 + + This commit adds an assumption to two test methods in + SSLTrustRestrictionsTests that we are not on JDK 11 as the tests + currently fail there. + + Relates #29989 + +commit b42765c82fa561506f12c0363b2e30e73d352497 +Author: Vladimir Dolzhenko +Date: Tue Jul 31 16:21:48 2018 +0200 + + Increase max chunk size to 256Mb for repo-azure (#32101) + + Increase max chunk size to 256Mb for repo-azure + +commit 736d2e8a65bf314286b53223753b6749e785725c +Author: Alexander Reelsen +Date: Tue Jul 31 16:14:37 2018 +0200 + + Docs: Fix README upgrade mention (#32313) + + The README still mentioned Elasticsearch 1.x. This commit links to the + documentation instead and properly formats the last paragraph. + +commit 4fa92cbf49326b45509ce2593f962f276282dca9 +Author: Sohaib Iftikhar +Date: Tue Jul 31 16:11:17 2018 +0200 + + Changed ReindexRequest to use Writeable.Reader (#32401) + + -- This is a pre-stage for adding the reindex API to the REST high-level-client + -- Follows the pattern set in #26315 + +commit 6e9f33843d1914be09c5cc966bac6a313d9a27bb +Author: Tal Levy +Date: Tue Jul 31 07:06:52 2018 -0700 + + stop throwing an IllegalStateException for unrecognized steps (#32212) + + Since the reason for a step not being found in a registry may be due to staleness of the + registry between it and the cluster state, we do not want to throw an IllegalStateException. + + Staleness is something that will be self-healing after follow-up applications of the cluster state + updates, so this is a recoverable issue that should log a warning instead of throwing an exception + + Closes #32181. + +commit adb93da974f88d7939eeac2fbe9066d4555ca586 +Author: Daniel Mitterdorfer +Date: Tue Jul 31 15:04:07 2018 +0200 + + Mute KerberosAuthenticationIT + + Relates #32498 + +commit 6f93911955dfe0e694110271cb9dd0ab2991240d +Author: Paul Sanwald +Date: Tue Jul 31 08:52:16 2018 -0400 + + Fix AutoIntervalDateHistogram.testReduce random failures (#32301) + + 1. Refactor the test to use the same roundings as the implementation. + 2. Refactor the test verification logic to use `innerIntervals` when rounding. + +commit 97b379e0d4fdcf04c769788644676f9555e1f672 +Author: Colm O'Shea +Date: Tue Jul 31 13:33:23 2018 +0100 + + fix no=>not typo (#32463) + + Found a tiny typo while reading the docs + +commit 9703d06321913dab9e59a91b5e4aef41fede6b29 +Author: Daniel Mitterdorfer +Date: Tue Jul 31 13:28:49 2018 +0200 + + Mute QueryProfilerIT#testProfileMatchesRegular() + + Relates #32492 + +commit 11f6983d6919a694e0c8c623b3f711690a373006 +Author: Yannick Welsch +Date: Tue Jul 31 10:29:22 2018 +0200 + + HLRC: Add delete watch action (#32337) + + Adds the "delete watch" API to the High-Level Rest Client. + + Relates #29827 + +commit a3b272966d8de964251186cb06967e5ec7e66f89 +Author: Luca Cavanna +Date: Tue Jul 31 09:41:51 2018 +0200 + + High-level client: fix clusterAlias parsing in SearchHit (#32465) + + When using cross-cluster search through the high-level REST client, the cluster alias from each search hit was not parsed correctly. It would be part of the index field initially, but overridden just a few lines later once setting the shard target (in case we have enough info to build it from the response). In any case, getClusterAlias returns `null` which is a bug. + + With this change we rather parse back clusterAliases from the index name, set its corresponding field and properly handle the two possible cases depending on whether we can or cannot build the shard target object. + +commit 8b57e2e5ba7de60f07d411fec0300cb2178790ba +Author: David Turner +Date: Tue Jul 31 08:25:21 2018 +0100 + + Fix calculation of orientation of polygons (#27967) + + The method for working out whether a polygon is clockwise or anticlockwise is + mostly correct but doesn't work in some rare cases such as the included test + case. This commit fixes that. + +commit d4ea440e37dcc870c0a9ba63a2c005ae974729b2 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jul 31 11:18:08 2018 +1000 + + [Kerberos] Add missing javadocs (#32469) + + This commit adds missing javadocs and fixes few where + the build failed when using JDK 11 for compilation. + + Closes#32461 + +commit f0b36679eca9653c3358842b137fdf354a328901 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jul 31 10:59:36 2018 +1000 + + [Kerberos] Remove Kerberos bootstrap checks (#32451) + + This commit removes Kerberos bootstrap checks as they were more + validation checks and better done in Kerberos realm constructor + than as bootstrap checks. This also moves the check + for one Kerberos realm per node to where we initialize realms. + This commit adds few validations which were missing earlier + like missing read permissions on keytab file or if it is directory + to throw exception with error message. + +commit 8cfbb64d6e7286e12d9ef849591c2443e84d6367 +Author: Nhat Nguyen +Date: Mon Jul 30 20:53:09 2018 -0400 + + ShardFollowNodeTask should fetch operation once (#32455) + + Today ShardFollowNodeTask might fetch some operations more than once. + This happens because we ask the leading for up to max_batch_count + operations (instead of the left-over size) for the left-over request. + The leading then can freely respond up to the max_batch_count, and at + the same time, if one of the previous requests completed, we might issue + another read request whose range overlaps with the response of the + left-over request. + + Closes #32453 + +commit d75efbcf6839a63e05ae3b88c84b6324921f25b3 +Author: Tim Vernum +Date: Tue Jul 31 09:07:47 2018 +1000 + + Make get all app privs requires "*" permission (#32460) + + The default behaviour for "GetPrivileges" is to get all application + privileges. This should only be allowed if the user has access to + the "*" application. + +commit a314efc92004d78279af15afd3b1e3f54e5e6408 +Author: Lee Hinman +Date: Mon Jul 30 16:57:25 2018 -0600 + + Add high level rest client support for SetIndexLifecyclePolicy (#32443) + + This adds HLRC support for the ILM operation of setting an index's lifecycle + policy. + + It also includes extracting and renaming a number of classes (like the request + and response objects) as well as the addition of a new `IndexLifecycleClient` + for the HLRC. This is a prerequisite to making the `index.lifecycle.name` + setting internal only, because we require a dedicated REST endpoint to change + the policy, and our tests currently set this setting with the REST client + multiple places. A subsequent PR will change the setting to be internal and move + those uses over to this new API. + + This misses some links to the documentation because I don't think ILM has any + documentation available yet. + + Relates to #29827 and #29823 + +commit 4101fc4e3d0bd3ccc64c99e1bdd93dee6d8df4d8 +Author: Nik Everett +Date: Mon Jul 30 18:16:26 2018 -0400 + + Switch security to new style Requests (#32290) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/plugin/security` project to use the new + versions. + +commit 670630948b4831704fbaf4821554d4ca52a5a42d +Author: Nik Everett +Date: Mon Jul 30 18:07:49 2018 -0400 + + Switch security spi example to new style Requests (#32341) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/qa/security-example-spi-extension` + project to use the new versions. + +commit c69e62d96fef43e526bd130acc429a5a8536cc56 +Author: Jack Conradson +Date: Mon Jul 30 14:46:24 2018 -0700 + + Painless: Add PainlessConstructor (#32447) + + PainlessMethod was being used as both a method and a constructor, and while there are + similarities, there are also some major differences. This allows the reflection objects to be + stored reducing the number of other pieces of data stored in a PainlessMethod as they are + now redundant. This temporarily increases some of the code in FunctionRef and + PainlessDocGenerator as they now differentiate between constructors and methods, BUT + is also makes the code more maintainable because there aren't checks in several places + anymore to differentiate. + +commit 1e0fcebfe1227401343e879cb07fd212c20fe99e +Author: Tal Levy +Date: Mon Jul 30 14:32:55 2018 -0700 + + update rollover to leverage write-alias semantics (#32216) + + Rollover should not swap aliases when `is_write_index` is set to `true`. + Instead, both the new and old indices should have the rollover alias, + with the newly created index as the new write index + + Updates Rollover to leverage the ability to preserve aliases and swap which is the write index. + + Historically, Rollover would swap which index had the designated alias for writing documents against. This required users to keep a separate read-alias that enabled reading against both rolled over and newly created indices, whiles the write-alias was being re-assigned at every rollover. + + With the ability for aliases to designate a write index, Rollover can be a bit more flexible with its use of aliases. + + Updates include: + + - Rollover validates that the target alias has a write index (the index that is being rolled over). This means that the restriction that aliases only point to one index is no longer necessary. + - Rollover explicitly (and atomically) swaps which index is the write-index by explicitly assigning the existing index to have `is_write_index: false` and have the newly created index have its rollover alias as `is_write_index: true`. This is only done when `is_write_index: true` on the write index. Default behavior of removing the alias from the rolled over index stays when `is_write_index` is not explicitly set + + Relevant things that are staying the same: + + - Rollover is rejected if there exist any templates that match the newly-created index and configure the rollover-alias + - I think this existed to prevent the situation where an alias pointed to two indices for a short while. Although this can technically be relaxed, the specific cases that are safe are really particular and difficult to reason, so leaving the broad restriction sounds good + +commit dc97051dd5904a547618af3a063505ef41c0fcc7 +Author: Lee Hinman +Date: Mon Jul 30 14:23:04 2018 -0600 + + Make various LifecycleSettings Settings internal (#32381) + + These are only ever set internally during regular ILM execution, they don't need + to be set otherwise. + + A subsequent PR will work on adding a dedicated endpoint for the + `LIFECYCLE_NAME` setting so it can be changed by a user (and then marked as + `InternalIndex` as well) + + Relates to #29823 + +commit 072c0be8af0bdbe9e035e4da6bd8e3831fe1796a +Author: w-bonelli +Date: Mon Jul 30 15:48:14 2018 -0400 + + Update Fuzzy Query docs to clarify default behavior re max_expansions (#30819) + + Stating that the Fuzzy Query generates "all possible" matching terms is misleading, given that the query's default behavior is to generate a maximum of 50 matching terms. + + (cherry picked from commit 345a0071a2a41fd7f80ae9ef8a39a2cb4991aedd) + +commit cf7489899a957764404769a4b792cdbefbf6a729 +Author: Armin Braun +Date: Mon Jul 30 21:25:30 2018 +0200 + + INGEST: Clean up Java8 Stream Usage (#32059) + + * GrokProcessor: Rationalize the loop over the map to save allocations and indirection + * IngestDocument: Rationalize way we append to `List` + +commit c2e3bebab997367520d18761288ffea5a9a715f1 +Author: Ioannis Kakavas +Date: Mon Jul 30 22:15:59 2018 +0300 + + Ensure KeyStoreWrapper decryption exceptions are handled (#32464) + + * Ensure decryption related exceptions are handled + + This commit ensures that all possible Exceptions in + KeyStoreWrapper#decrypt() are handled. More specifically, in the + case that a wrong password is used for secure settings, calling readX + on the DataInputStream that wraps the CipherInputStream can throw an + IOException. It also adds a test for loading a KeyStoreWrapper with + a wrong password. + + Resolves #32411 + +commit 1fdc3f08be0a2f93b64e13794574e85c8993357c +Author: Nhat Nguyen +Date: Mon Jul 30 14:30:47 2018 -0400 + + Do not expose hard-deleted docs in Lucene history (#32333) + + Today when reading operation history in Lucene, we read all documents. + However, if indexing a document is aborted, IndexWriter will hard-delete + it; we, therefore, need to exclude that document from Lucene history. + + This commit makes sure that we exclude aborted documents by using the + hard liveDocs of a SegmentReader if there are deletes. + + Closes #32269 + +commit 2245812ef79b43534043e4b68a5cdbf3cec888fc +Merge: aa3b6e098c8 34d006f82a3 +Author: Nhat Nguyen +Date: Mon Jul 30 14:02:58 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Tests: Fix convert error tests to use fixed value (#32415) + IndicesClusterStateService should replace an init. replica with an init. primary with the same aId (#32374) + REST high-level client: parse back _ignored meta field (#32362) + [CI] Mute DocumentSubsetReaderTests testSearch + +commit 34d006f82a3061f56c76266be2971a7db3bde4c6 +Author: Ryan Ernst +Date: Mon Jul 30 10:00:55 2018 -0700 + + Tests: Fix convert error tests to use fixed value (#32415) + + The error tests for hex values previously used a random string of + digits, but this could be a valid hex value. This commit changes these + tests to use a fixed invalid hex value. + + closes #32370 + +commit 0cae19c8d7d373b3138ca5a73adbebf442e2abef +Author: Boaz Leskes +Date: Mon Jul 30 16:24:41 2018 +0300 + + IndicesClusterStateService should replace an init. replica with an init. primary with the same aId (#32374) + + In rare cases it is possible that a nodes gets an instruction to replace a replica + shard that's in `POST_RECOVERY` with a new initializing primary with the same allocation id. + This can happen by batching cluster states that include the starting of the replica, with + closing of the indices, opening it up again and allocating the primary shard to the node in + question. The node should then clean it's initializing replica and replace it with a new + initializing primary. + + I'm not sure whether the test I added really adds enough value as existing tests found this. The main reason I added is to allow for simpler reproduction and to double check I fixed it. I'm open to discuss if we should keep. + + Closes #32308 + +commit 9a4d0069f6e179cb0db0578c3bfcdf16f6e0e6a2 +Author: Luca Cavanna +Date: Mon Jul 30 13:43:40 2018 +0200 + + REST high-level client: parse back _ignored meta field (#32362) + + `GetResult` and `SearchHit` have been adjusted to parse back the `_ignored` meta field whenever it gets printed out. Expanded the existing tests to make sure this is covered. Fixed also a small problem around highlighted fields in `SearchHitTests`. + +commit 59bb37b5dac87a72fbae21da4d4925a459b37477 +Merge: 0682b4bbc97 14a4a740ace +Author: Colin Goodheart-Smithe +Date: Mon Jul 30 11:48:54 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 0682b4bbc9707030fdff3e4cf39cd6a306938e52 +Author: Colin Goodheart-Smithe +Date: Mon Jul 30 09:08:30 2018 +0100 + + Removes redundent NORELEASES and ObjectParserUtils (#32427) + +commit 14a4a740ace4c96cf02f79af145c1ee7ec2401e8 +Author: David Roberts +Date: Mon Jul 30 09:04:17 2018 +0100 + + [CI] Mute DocumentSubsetReaderTests testSearch + + Relates #32457 + +commit aa3b6e098c8875a09e73426f8bee4cd403caa33a +Author: Nhat Nguyen +Date: Sun Jul 29 21:57:45 2018 -0400 + + Reject follow request if following setting not enabled on follower (#32448) + + Today we do not check if the `following_index` setting of the follower + is enabled or not when processing a follow-request. If that setting is + disabled, the follower will use the default engine, not the following + engine. This change checks and rejects such invalid follow requests. + + Relates #30086 + +commit d2a88f5c62d3f399744572c243a1b4b8e878fc19 +Merge: a538b76f6f1 5b1ad8099bf +Author: Nhat Nguyen +Date: Sat Jul 28 07:50:39 2018 -0400 + + Merge branch 'master' into ccr + + * master: + TEST: testDocStats should always use forceMerge (#32450) + TEST: Avoid deletion in FlushIT + AwaitsFix IndexShardTests#testDocStats + Painless: Add method type to method. (#32441) + +commit 5b1ad8099bfd0738bb16331e6f6952b8726b849e +Author: Nhat Nguyen +Date: Sat Jul 28 07:41:30 2018 -0400 + + TEST: testDocStats should always use forceMerge (#32450) + + Due to the recent change in LUCENE-8263, we need to adjust the deletion + ration to between 10% to 33% to preserve the current behavior of the + test. However, we may need another refinement if soft-deletes is enabled + as the actual deletes are different because of delete tombstones. + + This commit prefers to always execute forceMerge instead of adjusting + the deletion ratio so that this test can focus on testing docStats. + + Closes #32449 + +commit a538b76f6f1bb486aff903dce5f1511de868c5bb +Author: Nhat Nguyen +Date: Fri Jul 27 23:44:35 2018 -0400 + + TEST: avoid merge in testSegmentMemoryTrackedInBreaker + + This commit indexes an extra document to avoid triggering merges. + + Relates LUCENE-8263 + +commit 6e98615cc1b324f9c2f543125c4cc7a3d01f3591 +Author: Nhat Nguyen +Date: Fri Jul 27 23:14:24 2018 -0400 + + TEST: Avoid deletion in FlushIT + + Due to the recent change in LUCENE-8263, a merge can be triggered if the + deletion ration is higher than 33%. An in-progress merge can prevent a + synced-flush from issuing. + + This commit avoids deletes by using different docIds. + + Closes #32436 + +commit 139631c77d1a6bac002f82952d599a386122b1e3 +Author: Nhat Nguyen +Date: Fri Jul 27 20:48:17 2018 -0400 + + AwaitsFix IndexShardTests#testDocStats + + Relates #32449 + +commit e9e1095596ace4cd50c5a09a6860506add1eb216 +Author: Jack Conradson +Date: Fri Jul 27 14:23:37 2018 -0700 + + Painless: Add method type to method. (#32441) + + MethodType can be computed at compile-time rather than run-time. This removes the + method that collects MethodType at run-time from a PainlessMethod since is it no longer + necessary. + +commit 2f756b00f6577ead26ef64ac12fa59a4fd5388d8 +Merge: 8474f8a01c8 588db621ac0 +Author: Nhat Nguyen +Date: Fri Jul 27 16:23:58 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Remove reference to non-existent store type (#32418) + [TEST] Mute failing FlushIT test + Fix ordering of bootstrap checks in docs (#32417) + [TEST] Mute failing InternalEngineTests#testSeqNoAndCheckpoints + [TEST] Mute failing testConvertLongHexError + bump lucene version after backport + Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390) + [Kerberos] Avoid vagrant update on precommit (#32416) + TESTS: Move netty leak detection to paranoid level (#32354) + [DOCS] Fixes formatting of scope object in job resource + Copy missing segment attributes in getSegmentInfo (#32396) + AbstractQueryTestCase should run without type less often (#28936) + INGEST: Fix Deprecation Warning in Script Proc. (#32407) + Switch x-pack/plugin to new style Requests (#32327) + Docs: Correcting a typo in tophits (#32359) + Build: Stop double generating buildSrc pom (#32408) + TEST: Avoid triggering merges in FlushIT + Fix missing JavaDoc for @throws in several places in KerberosTicketValidator. + Switch x-pack full restart to new style Requests (#32294) + Release requests in cors handler (#32364) + Painless: Clean Up PainlessClass Variables (#32380) + Docs: Fix callouts in put license HL REST docs (#32363) + [ML] Consistent pattern for strict/lenient parser names (#32399) + Update update-settings.asciidoc (#31378) + Remove some dead code (#31993) + Introduce index store plugins (#32375) + Rank-Eval: Reduce scope of an unchecked supression + Make sure _forcemerge respects `max_num_segments`. (#32291) + TESTS: Fix Buf Leaks in HttpReadWriteHandlerTests (#32377) + Only enforce password hashing check if FIPS enabled (#32383) + +commit 588db621ac089cdb193f0500078e736e7d36d6ce +Author: Jason Tedor +Date: Fri Jul 27 11:24:03 2018 -0400 + + Remove reference to non-existent store type (#32418) + + We removed the default_fs store type yet the docs still contain a + reference to them. This commit addresses that by removing this + reference, and changing a reference to this section of the docs to + instead refer to mmapfs. + +commit dcb5d24639ce073d354702c8fe1f038ce9f948bb +Author: javanna +Date: Fri Jul 27 17:08:54 2018 +0200 + + [TEST] Mute failing FlushIT test + + See #32436 + +commit 3ac57f0ba37aa2cdf24cc85d8037ffc22816cf6a +Author: Jason Tedor +Date: Fri Jul 27 10:40:16 2018 -0400 + + Fix ordering of bootstrap checks in docs (#32417) + + In the section of the bootstrap checks docs for the maximum map count + check, we refer to max size virtual memory check and explicitly call out + the maximum size virtual memory check as being the previous + point. However, this is not correct as the previous point is currently + the max file size check. It does make sense for these two checks to be + proximate to each other in the docs so this commit reorders the checks + so that the maximum size virtual memory check indeed comes before the + maximum map count check. This makes the sense in the maximum map count + check correct. + +commit 7aa53654976ce9c9932d36dca8ed81a342cce70b +Author: javanna +Date: Fri Jul 27 14:41:32 2018 +0200 + + [TEST] Mute failing InternalEngineTests#testSeqNoAndCheckpoints + +commit 783e427d36f37258793c535a8a4b0d765da55f79 +Author: Colin Goodheart-Smithe +Date: Fri Jul 27 13:40:10 2018 +0100 + + Skips to next available action on missing step (#32283) + + if policy update on index means current step no longer exists + + This change only updates the setPolicy for index to add this + functionality. The update policy API will be changed in a follow up PR. + +commit b209f956262fb516b543471a9b131afa7ba8096d +Author: Colin Goodheart-Smithe +Date: Fri Jul 27 13:20:38 2018 +0100 + + Fix serialisation bug + +commit 8474f8a01c8fc26ca78cd532c28781bcd56b15a2 +Author: Nhat Nguyen +Date: Fri Jul 27 08:16:52 2018 -0400 + + Validate source of an index in LuceneChangesSnapshot (#32288) + + Today it's possible to encounter an Index operation in Lucene whose + _source is disabled, and _recovery_source was pruned by the MergePolicy. + If it's the case, we create a Translog#Index without source and let the + caller validate it later. However, this approach is challenging for the + caller. + + Deletes and No-Ops don't allow invoking "source()" method. The caller + has to make sure to call "source()" only on index operations. The + current implementation in CCR does not follow this and fail to replica + deletes or no-ops. Moreover, it's easier to reason if a Translog#Index + always has the source. + +commit dfb277cfb7969ad543a31dea234579d0e886a73a +Merge: 99e8b5bc13c 5decb23687e +Author: Colin Goodheart-Smithe +Date: Fri Jul 27 10:51:09 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 83d007e7bee9763ab4b15553110d1b7a829f614b +Author: javanna +Date: Fri Jul 27 11:50:13 2018 +0200 + + [TEST] Mute failing testConvertLongHexError + + See #32370 + +commit 5decb23687eb19ed1d229c19382897f5010fb5ed +Author: Jim Ferenczi +Date: Fri Jul 27 10:50:22 2018 +0200 + + bump lucene version after backport + +commit 53ff06e621213cb007d0e03b8ec8d80431d9186a +Author: Jim Ferenczi +Date: Fri Jul 27 08:28:51 2018 +0200 + + Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390) + + The main highlight is the removal of the reclaim_deletes_weight in the TieredMergePolicy. + The es setting index.merge.policy.reclaim_deletes_weight is deprecated in this commit and the value is ignored. The new merge policy setting setDeletesPctAllowed should be added in a follow up. + +commit 9198bcf78e4ff75ddefeceebb7b139c9485e2b4c +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Jul 27 12:01:52 2018 +1000 + + [Kerberos] Avoid vagrant update on precommit (#32416) + + This commit avoids dependency during compile on copy keytab to + be present in the generated sources so pre-commit does not + stall for updating vagrant box. + + Closes#32387 + +commit 90c58872ff4d1fcd72c79adc999d6713309f851d +Author: Nhat Nguyen +Date: Thu Jul 26 21:02:43 2018 -0400 + + Only enable soft-deletes in 6.5 or later + +commit 1628c833c7da15748011e17ba03c2bc51e312428 +Author: Armin Braun +Date: Thu Jul 26 21:36:49 2018 +0200 + + TESTS: Move netty leak detection to paranoid level (#32354) + +commit 67a884ec88bd18a7866a82efe7b164b3b48459d2 +Author: lcawl +Date: Thu Jul 26 12:28:26 2018 -0700 + + [DOCS] Fixes formatting of scope object in job resource + +commit 860f92fcdd46d5e91faf3c72d4fd7732b43f46b5 +Author: Jim Ferenczi +Date: Thu Jul 26 20:29:27 2018 +0200 + + Copy missing segment attributes in getSegmentInfo (#32396) + + The index sort and the attributes map of a segment are not copied + on committed segments that are not loaded by the internal or external searcher. + +commit 8e5f281b27ccef436ad687b9ccb19b271b445ce9 +Author: Jim Ferenczi +Date: Thu Jul 26 20:29:05 2018 +0200 + + AbstractQueryTestCase should run without type less often (#28936) + + This commit changes the randomization to always create an index with a type. + It also adds a way to create a query shard context that maps to an index with + no type registered in order to explicitely test cases where there is no type. + +commit 57876bfeb916e18b57e0624a3dfed9779b109f14 +Author: Armin Braun +Date: Thu Jul 26 19:55:28 2018 +0200 + + INGEST: Fix Deprecation Warning in Script Proc. (#32407) + + * Using short script form normalized to a map that used 'inline' instead of 'source' so a short form processor definition like: + + ``` + { + "script": "ctx.foo= 'bar'" + } + ``` + + would always warn about the following deprecation: + + ``` + #! Deprecation: Deprecated field [inline] used, expected [source] + ``` + +commit 643235d46a80dd0201677fa685ed14c8a5a7c6de +Author: Nik Everett +Date: Thu Jul 26 13:31:16 2018 -0400 + + Switch x-pack/plugin to new style Requests (#32327) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/plugin` project to use the new versions. + +commit 7ad16ffd84046a200e0b7ef9a8ac0388e4d62f74 +Author: Sandeep Kanabar +Date: Thu Jul 26 22:59:17 2018 +0530 + + Docs: Correcting a typo in tophits (#32359) + +commit 0f783ce94b5f670e9877c37ba481d5c2f1e4d7ac +Author: Nik Everett +Date: Thu Jul 26 13:19:05 2018 -0400 + + Build: Stop double generating buildSrc pom (#32408) + + When we added the `java-gradle-plugin` to `buildSrc` it added a second + task to generate the pom that duplicates the publishing work that we + configure in `BuildPlugin`. Not only does it dupliciate the pom, it + creates a pom that is missing things like `name` and `description` which + are required for publishing to maven central. + + This change disables the duplicate pom generation. + +commit 0ed3458534beb8d93852e57b2ee422324613c528 +Author: Nhat Nguyen +Date: Thu Jul 26 12:35:15 2018 -0400 + + TEST: Avoid triggering merges in FlushIT + + In testSyncedFlushSkipOutOfSyncReplicas, we reindex the extra documents + to all shards including the out-of-sync replica. However, reindexing to + that replica can trigger merges (due to the new deletes) which cause the + synced-flush failed. This test starts failing after we aggressively + trigger merges segments with a large number of deletes in LUCENE-8263. + +commit eb73dde7c83f1e988104c2539763768d57c1454e +Author: Jack Conradson +Date: Thu Jul 26 09:14:43 2018 -0700 + + Fix missing JavaDoc for @throws in several places in + KerberosTicketValidator. + +commit 11a2189c6f738fe0d2773eba49c6feca8e377dfa +Author: Nik Everett +Date: Thu Jul 26 12:11:31 2018 -0400 + + Switch x-pack full restart to new style Requests (#32294) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack:qa:full-cluster-restart` project to use + the new versions. + +commit 7a56df7c98918d78454a7da8bfe1795227b6c8e2 +Author: Tim Brooks +Date: Thu Jul 26 10:06:24 2018 -0600 + + Release requests in cors handler (#32364) + + There are two scenarios where a http request could terminate in the cors + handler. If that occurs, the requests need to be released. This commit + releases those requests. + +commit df579f8bce005601d8b6c24f91699e67a92eac19 +Author: Jack Conradson +Date: Thu Jul 26 09:02:06 2018 -0700 + + Painless: Clean Up PainlessClass Variables (#32380) + + Removes the variables name, clazz, and type as they are unnecessary. Renames + staticMembers -> staticFields, members -> fields, getters -> getterMethodHandles, and + setters -> setterMethodHandles. + +commit 3b050194f9f014735dea82bd96fc63efb43d5cc7 +Author: Igor Motov +Date: Thu Jul 26 09:00:35 2018 -0700 + + Docs: Fix callouts in put license HL REST docs (#32363) + +commit 0afa265ac90277b0b21dc125da29181550b7e99e +Author: David Roberts +Date: Thu Jul 26 16:55:40 2018 +0100 + + [ML] Consistent pattern for strict/lenient parser names (#32399) + + Previously we had two patterns for naming of strict + and lenient parsers. + + Some classes had CONFIG_PARSER and METADATA_PARSER, + and used an enum to pass the parser type to nested + parsers. + + Other classes had STRICT_PARSER and LENIENT_PARSER + and used ternary operators to pass the parser type + to nested parsers. + + This change makes all ML classes use the second of + the patterns described above. + +commit 63a043676461d904b7447a1f494a1243ed154994 +Author: mzbeck <39967352+mzbeck@users.noreply.github.com> +Date: Thu Jul 26 08:15:10 2018 -0700 + + Update update-settings.asciidoc (#31378) + +commit 35ae87125d5fb5ff3395e43ada49b8a81cdf9a3e +Author: Christoph Büscher +Date: Thu Jul 26 17:12:51 2018 +0200 + + Remove some dead code (#31993) + + Removing some dead code or supressing warnings where apropriate. Most of the + time the variable tested for null is dereferenced earlier or never used before. + +commit eb675a1c4d2660c7f0090595d0771809245030da +Author: Jason Tedor +Date: Thu Jul 26 08:05:49 2018 -0400 + + Introduce index store plugins (#32375) + + Today we allow plugins to add index store implementations yet we are not + doing this in our new way of managing plugins as pull versus push. That + is, today we still allow plugins to push index store providers via an on + index module call where they can turn around and add an index + store. Aside from being inconsistent with how we manage plugins today + where we would look to pull such implementations from plugins at node + creation time, it also means that we do not know at a top-level (for + example, in the indices service) which index stores are available. This + commit addresses this by adding a dedicated plugin type for index store + plugins, removing the index module hook for adding index stores, and by + aggregating these into the top-level of the indices service. + +commit 99e8b5bc13c5a2725ecd31923c47edb8e0604d7b +Merge: 3daefe66af6 bec888fa78e +Author: Colin Goodheart-Smithe +Date: Thu Jul 26 10:49:57 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit bec888fa78e67d2221af2962bbca8e6ed0d9cbfa +Author: Christoph Büscher +Date: Thu Jul 26 11:15:53 2018 +0200 + + Rank-Eval: Reduce scope of an unchecked supression + + We should only supress the unchecked warnings on ConstructingObjectParser. + +commit f7ba14d5f478aca194e8f56bca9fd63cf6010996 +Author: Adrien Grand +Date: Thu Jul 26 09:02:58 2018 +0200 + + Make sure _forcemerge respects `max_num_segments`. (#32291) + + An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976) + will make TieredMergePolicy respect the maximum merged segment size all the + time, meaning it will possibly not respect the `max_num_segments` parameter + anymore if the shard is larger than the maximum segment size. + + This change makes sure that `max_num_segments` is respected for now in order + to give us time to think about how to integrate this change, and also to delay + it until 7.0 as this might be a big-enough change for us to wait for a new + major version. + +commit 48885d2b7e0fd675ab6db791c9c3e021efcb2f2d +Author: Armin Braun +Date: Thu Jul 26 07:01:59 2018 +0200 + + TESTS: Fix Buf Leaks in HttpReadWriteHandlerTests (#32377) + + * TESTS: Fix Buf Leaks in HttpReadWriteHandlerTests + + * Release all ref counted things that weren't getting properly released + * Mannually force channel promise to be completed because mock channel doesn't do it and it prevents one `release` call in `io.netty.channel.ChannelOutboundHandlerAdapter#write` from firing + +commit 467a60ba0c7b5cde5275d9a18042a3df5a40b1ab +Author: Jason Tedor +Date: Wed Jul 25 20:57:03 2018 -0400 + + Only enforce password hashing check if FIPS enabled (#32383) + + This commit modifies the FIPS password hashing algorithm check to only + be executed if FIPS mode is enabled. + +commit caefed4d337b33dae32b940b210134f8ef9819dc +Merge: b8548973e3a a577fb3381c +Author: Nhat Nguyen +Date: Wed Jul 25 20:12:13 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [DOCS] Fix formatting error in Slack action + Painless: Fix documentation links to use existing refs (#32335) + Painless: Decouple PainlessLookupBuilder and Whitelists (#32346) + [DOCS] Adds recommendation for xpack.security.enabled (#32345) + [TEST] Mute ConvertProcessortTests.testConvertIntHexError + [TEST] Fix failure due to exception message in java11 (#32321) + [DOCS] Fixes typo in ML aggregations page + [DOCS] Adds link from bucket_span property to common time units + [ML][DOCS] Add documentation for detector rules and filters (#32013) + Add opaque_id to index audit logging (#32260) + Add 6.5.0 version to master + fixes broken build for third-party-tests (#32353) + +commit a577fb3381c8c9229502677e8d403f5fdd7c0704 +Author: lcawl +Date: Wed Jul 25 14:19:19 2018 -0700 + + [DOCS] Fix formatting error in Slack action + +commit 3daefe66af6f6246d4c9fb625d65d75ffc76cd4a +Author: Lee Hinman +Date: Wed Jul 25 14:11:14 2018 -0600 + + Remove "best_compression" option from the ForceMergeAction (#32373) + + This option is only settable while the index is closed, and doesn't make sense + for a force merge. + + Relates to #29823 + +commit 10bfedeb530f2ad082ddaf74a8efc3fc45644b1b +Author: Jack Conradson +Date: Wed Jul 25 10:59:25 2018 -0700 + + Painless: Fix documentation links to use existing refs (#32335) + + Uses ref and xpack-ref instead of custom ones in the Painless docs for + long-term maintainability. + +commit 853aa0afb4b9ac38d30a32ba93f047990367ac44 +Author: Jack Conradson +Date: Wed Jul 25 10:52:01 2018 -0700 + + Painless: Decouple PainlessLookupBuilder and Whitelists (#32346) + + Implements a static function in PainlessLookupBuilder that contains all the logic related + to Whitelist. PainlessLookupBuilder is available for use in loading from methods beyond + Whitelist now. + +commit b8548973e3a11e8323950eacc9264909377725f6 +Author: Jason Tedor +Date: Wed Jul 25 13:13:53 2018 -0400 + + Maintain ordering in X-Pack project substitutions + + This commit fixes the ordering of the project substitutions for + X-Pack. This is not a bug, simply keeping them in order helps maintain + them. + +commit a75c6a2f57fe4e1e138016da4963d499ee3fdfae +Author: Jason Tedor +Date: Wed Jul 25 13:05:35 2018 -0400 + + Fix index-lifecycle build + + We have to use the shadow configuration now when packaging + index-lifecycle or we duplicate the X-Pack protocol library from core. + +commit 8067250fbe2eed5395f0ed11cac4087bb07053a7 +Author: Lisa Cawley +Date: Wed Jul 25 09:51:40 2018 -0700 + + [DOCS] Adds recommendation for xpack.security.enabled (#32345) + +commit de53f0123f99b4cc002f465adcf03142534111b3 +Author: Dimitris Athanasiou +Date: Wed Jul 25 17:35:23 2018 +0100 + + [TEST] Mute ConvertProcessortTests.testConvertIntHexError + + Relates #32370 + +commit 86928e7d2478ae766be2bf95c42d096160a8e0dc +Merge: 81faa2fe6b7 6cf7588c3d8 +Author: Tal Levy +Date: Wed Jul 25 09:01:35 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 6cf7588c3d8e7d79c77c9fd3fd091bc7c11f512d +Author: Zachary Tong +Date: Wed Jul 25 11:34:26 2018 -0400 + + [TEST] Fix failure due to exception message in java11 (#32321) + + Java 11 uses more verbose exceptions messages, causing this assertion + to fail. Changed the test to be less restrictive and only look + for the classes we care about. + +commit 5a12c63b4e98c9ef95baa6cf2e932837e7031420 +Author: lcawl +Date: Tue Jul 24 16:11:56 2018 -0700 + + [DOCS] Fixes typo in ML aggregations page + +commit 6832aa6797c3105747554d9e21213f7d2a15e344 +Author: lcawl +Date: Tue Jul 24 17:06:46 2018 -0700 + + [DOCS] Adds link from bucket_span property to common time units + +commit 9a7a649755c158977ec238920ed710b993ffceb4 +Author: Dimitris Athanasiou +Date: Wed Jul 25 16:10:32 2018 +0100 + + [ML][DOCS] Add documentation for detector rules and filters (#32013) + +commit e12e2e0cdda7745fcfb1570a706d1c8ede21d6e6 +Author: Igor Motov +Date: Wed Jul 25 08:07:09 2018 -0700 + + Add opaque_id to index audit logging (#32260) + + Logs opaque_id if it is available with all audit log messages using + index-based audit log. + + Closes #31521 + +commit 9efd1407d5875c34e9f661fd9743c8bd78155878 +Author: Dimitris Athanasiou +Date: Wed Jul 25 15:57:07 2018 +0100 + + Add 6.5.0 version to master + + This commit adds the 6.5.0 version constant to the master branch. + +commit a2547168d27f0aada27d824f0d7a596a1a2c4fa1 +Author: Vladimir Dolzhenko +Date: Wed Jul 25 16:34:30 2018 +0200 + + fixes broken build for third-party-tests (#32353) + + fixes broken build repository-s3 for third-party-tests + +commit 02c226820c6438ef6c8e273073ca09b9fb716be1 +Merge: d2d78415a94 6aea82956cb +Author: Jason Tedor +Date: Wed Jul 25 09:32:20 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: + Number of utilities for writing gradle integration tests (#32282) + Determine the minimum gradle version based on the wrapper (#32226) + Enable FIPS JVM in CI (#32330) + Fix JarHell on X-Pack protocol + +commit 81faa2fe6b73a251aa16c17fb6266d56b5205b2b +Merge: e5230306700 6aea82956cb +Author: Colin Goodheart-Smithe +Date: Wed Jul 25 10:39:14 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 6aea82956cb9b17481847d83d1881af3e96e3d52 +Author: Alpar Torok +Date: Wed Jul 25 06:22:54 2018 +0000 + + Number of utilities for writing gradle integration tests (#32282) + + These are collected from a number of open PRs and are required to + improove existing and write more readable future tests. + I am extracting them to their own PR hoping to be able to merge and use + them sooner. + +commit b6c14935d725bbe404707ef1d363d8c504a88155 +Author: Alpar Torok +Date: Wed Jul 25 06:21:03 2018 +0000 + + Determine the minimum gradle version based on the wrapper (#32226) + + * Determine the minimum gradle version based on the wrapper + + This is restrictive and forces users of the plugin to move together with + us, but without integration tests it's close to impossible to make sure + that the claimed compatability is really there. + If we do want to offer more flexibility, we should add those tests + first. + + * Track gradle version in individual file + + * PR review + +commit d0fd23669add7e72e0f344337dc66f6c1c43a2f1 +Author: Ioannis Kakavas +Date: Wed Jul 25 08:48:19 2018 +0300 + + Enable FIPS JVM in CI (#32330) + + Now that #31666 and #31989 are merged we can run our tests in + fips JVM. This commits enables us to run tests on a Java 8 + JVM using BouncyCastleFIPS as a security Provider. + +commit d2d78415a94aae8c5a37ae1167169b4641f1a274 +Author: Nik Everett +Date: Tue Jul 24 23:42:06 2018 -0400 + + Build: Fix jarHell error I caused by last backport + + I fixed this locally but didn't commit it.... Ooops. Shouldn't merge + late a night.... + +commit cd8b80da583a0b705b6406005343eea996fcc1e8 +Author: Nhat Nguyen +Date: Tue Jul 24 23:58:55 2018 -0400 + + Use shadow plugin in ccr/qa + +commit 33e3822c5ded4a36742c354608f9ce4220a61029 +Author: Tim Vernum +Date: Wed Jul 25 14:14:19 2018 +1000 + + Fix JarHell on X-Pack protocol + + Ports 466bc8c from 6.x + +commit a5d8f0b55a01bb041f444034e40128a8d9738ebb +Author: Nhat Nguyen +Date: Tue Jul 24 22:28:17 2018 -0400 + + CCR: use shadow plugin + + Relates #32240 + +commit ab4deefbe03132c1f62a47e91bbad0e7bf103c6f +Merge: 88190299df8 e43375bf9ab +Author: Nhat Nguyen +Date: Tue Jul 24 21:33:50 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Security: revert to old way of merging automata (#32254) + Networking: Fix test leaking buffer (#32296) + Undo a debugging change that snuck in during the field aliases merge. + Painless: Update More Methods to New Naming Scheme (#32305) + [TEST] Fix assumeFalse -> assumeTrue in SSLReloadIntegTests + Ingest: Support integer and long hex values in convert (#32213) + Introduce fips_mode setting and associated checks (#32326) + Add V_6_3_3 version constant + [DOCS] Removed extraneous callout number. + Rest HL client: Add put license action (#32214) + Add ERR to ranking evaluation documentation (#32314) + Introduce Application Privileges with support for Kibana RBAC (#32309) + Build: Shadow x-pack:protocol into x-pack:plugin:core (#32240) + [Kerberos] Add Kerberos authentication support (#32263) + [ML] Extract persistent task methods from MlMetadata (#32319) + Add Restore Snapshot High Level REST API + Register ERR metric with NamedXContentRegistry (#32320) + fixes broken build for third-party-tests (#32315) + Allow Integ Tests to run in a FIPS-140 JVM (#31989) + [DOCS] Rollup Caps API incorrectly mentions GET Jobs API (#32280) + awaitsfix testRandomClusterStateUpdates + [TEST] add version skip to weighted_avg tests + Consistent encoder names (#29492) + Add WeightedAvg metric aggregation (#31037) + Switch monitoring to new style Requests (#32255) + Rename ranking evaluation `quality_level` to `metric_score` (#32168) + Fix a test bug around nested aggregations and field aliases. (#32287) + Add new permission for JDK11 to load JAAS libraries (#32132) + Silence SSL reload test that fails on JDK 11 + [test] package pre-install java check (#32259) + specify subdirs of lib, bin, modules in package (#32253) + Switch x-pack:core to new style Requests (#32252) + awaitsfix SSLConfigurationReloaderTests + Painless: Clean up add methods in PainlessLookup (#32258) + Fail shard if IndexShard#storeStats runs into an IOException (#32241) + AwaitsFix RecoveryIT#testHistoryUUIDIsGenerated + Remove unnecessary warning supressions (#32250) + CCE when re-throwing "shard not available" exception in TransportShardMultiGetAction (#32185) + Add new fields to monitoring template for Beats state (#32085) + +commit e43375bf9abaa7825eec69831d5ba92047896586 +Author: Jay Modi +Date: Tue Jul 24 16:26:50 2018 -0600 + + Security: revert to old way of merging automata (#32254) + + This commit reverts to the pre-6.3 way of merging automata as the + change in 6.3 significantly impacts the performance for roles with a + large number of concrete indices. In addition, the maximum number of + states for security automata has been increased to 100,000 in order + to allow users to use roles that caused problems pre-6.3 and 6.3 fixed. + + As an escape hatch, the maximum number of states is configurable with + a setting so that users with complex patterns in roles can increase + the states with the knowledge that there is more memory usage. + +commit 717df26fc3e97f4142d3d56ba9fea816799081b0 +Author: Armin Braun +Date: Tue Jul 24 23:04:22 2018 +0200 + + Networking: Fix test leaking buffer (#32296) + + * Test `handler` must release buffer the same way the replaced `org.elasticsearch.http.netty4.Netty4HttpRequestHandler#channelRead0` releases it + * Closes #32289 + +commit 54ba3ea286478033705d354532351e486bdcd7b2 +Author: Julie Tibshirani +Date: Tue Jul 24 13:19:24 2018 -0700 + + Undo a debugging change that snuck in during the field aliases merge. + +commit 1690451a9fb3a049b39736b39ea02dbf535b1991 +Author: Jack Conradson +Date: Tue Jul 24 13:08:05 2018 -0700 + + Painless: Update More Methods to New Naming Scheme (#32305) + + This finishes the updating the methods in the PainlessLookupBuilder to the new naming scheme. Mechanical change. Methods include the ones used for copying members in the inheritance hierarchy, calculating shortcuts, and setting the functional interface. + +commit 73be9840dd086dc431d8edfdbfba3ea5b39711c2 +Author: Lee Hinman +Date: Tue Jul 24 13:10:23 2018 -0600 + + [TEST] Fix assumeFalse -> assumeTrue in SSLReloadIntegTests + +commit 49d4b26f166ff6657cc799d7d77b44a7d6cf9a5d +Author: Ryan Ernst +Date: Tue Jul 24 12:05:50 2018 -0700 + + Ingest: Support integer and long hex values in convert (#32213) + + This commit adds checks for hex formatted strings in the convert + processor, allowing strings like `0x1` to be parsed as integer `1`. + + closes #32182 + +commit be40a6982a9a90879fe198260df6a4cb3c0e8582 +Author: Ioannis Kakavas +Date: Tue Jul 24 22:05:00 2018 +0300 + + Introduce fips_mode setting and associated checks (#32326) + + * Introduce fips_mode setting and associated checks + + Introduce xpack.security.fips_mode.enabled setting ( default false) + When it is set to true, a number of Bootstrap checks are performed: + - Check that Secure Settings are of the latest version (3) + - Check that no JKS keystores are configured + - Check that compliant algorithms ( PBKDF2 family ) are used for + password hashing + +commit e0b7e4b1c5f13af43daebf37657206d4da380b92 +Author: Lee Hinman +Date: Tue Jul 24 10:54:21 2018 -0600 + + Add V_6_3_3 version constant + + Relates to #32329 + +commit 32eb4d41b655a8e24ade22a3ee99a3e176ed847e +Author: debadair +Date: Tue Jul 24 11:51:52 2018 -0700 + + [DOCS] Removed extraneous callout number. + +commit e5230306700005de598bef5d056dd5edaf9db234 +Merge: a2abc865959 29c802f88e9 +Author: Tal Levy +Date: Tue Jul 24 11:23:44 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 29c802f88e92232a2dc448dc4e91443e6e831148 +Author: Igor Motov +Date: Tue Jul 24 11:19:07 2018 -0700 + + Rest HL client: Add put license action (#32214) + + In the HL REST client we replace the License object with a string, because of + complexity of this class. It is also not really needed on the client side since + end-users are not interacting with the license besides passing it as a string + to the server. + + Relates #29827 + +commit c1cc0cef6136d0c92bcb132f5321c653df8544da +Author: Christoph Büscher +Date: Tue Jul 24 19:58:34 2018 +0200 + + Add ERR to ranking evaluation documentation (#32314) + + This change adds a section about the Expected Reciprocal Rank metric (ERR) to + the Ranking Evaluation documentation. + +commit 387c3c7f1d9c5aa6e5c02a71ed3a64ecf6c11872 +Author: Tim Vernum +Date: Wed Jul 25 02:34:46 2018 +1000 + + Introduce Application Privileges with support for Kibana RBAC (#32309) + + This commit introduces "Application Privileges" to the X-Pack security + model. + + Application Privileges are managed within Elasticsearch, and can be + tested with the _has_privileges API, but do not grant access to any + actions or resources within Elasticsearch. Their purpose is to allow + applications outside of Elasticsearch to represent and store their own + privileges model within Elasticsearch roles. + + Access to manage application privileges is handled in a new way that + grants permission to specific application names only. This lays the + foundation for more OLS on cluster privileges, which is implemented by + allowing a cluster permission to inspect not just the action being + executed, but also the request to which the action is applied. + To support this, a "conditional cluster privilege" is introduced, which + is like the existing cluster privilege, except that it has a Predicate + over the request as well as over the action name. + + Specifically, this adds + - GET/PUT/DELETE actions for defining application level privileges + - application privileges in role definitions + - application privileges in the has_privileges API + - changes to the cluster permission class to support checking of request + objects + - a new "global" element on role definition to provide cluster object + level security (only for manage application privileges) + - changes to `kibana_user`, `kibana_dashboard_only_user` and + `kibana_system` roles to use and manage application privileges + + Closes #29820 + Closes #31559 + +commit e6b9f59e4eb211c3c519864c3f195fa23c6fbc26 +Author: Nik Everett +Date: Tue Jul 24 11:53:04 2018 -0400 + + Build: Shadow x-pack:protocol into x-pack:plugin:core (#32240) + + This bundles the x-pack:protocol project into the x-pack:plugin:core + project because we'd like folks to consider it an implementation detail + of our build rather than a separate artifact to be managed and depended + on. It is now bundled into both x-pack:plugin:core and + client:rest-high-level. To make this work I had to fix a few things. + + Firstly, I had to make PluginBuildPlugin work with the shadow plugin. + In that case we have to bundle only the `shadow` dependencies and the + shadow jar. + + Secondly, every reference to x-pack:plugin:core has to use the `shadow` + configuration. Without that the reference is missing all of the + un-shadowed dependencies. I tried to make it so that applying the shadow + plugin automatically redefines the `default` configuration to mirror the + `shadow` configuration which would allow us to use bare project references + to the x-pack:plugin:core project but I couldn't make it work. It'd *look* + like it works but then fail for transitive dependencies anyway. I think + it is still a good thing to do but I don't have the willpower to do it + now. + + Finally, I had to fix an issue where Eclipse and IntelliJ didn't properly + reference shadowed transitive dependencies. Neither IDE supports shadowing + natively so they have to reference the shadowed projects. We fix this by + detecting `shadow` dependencies when in "Intellij mode" or "Eclipse mode" + and adding `runtime` dependencies to the same target. This convinces + IntelliJ and Eclipse to play nice. + +commit a525c36c6092de16a0260ffe3c5b5b2e705f65c4 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Wed Jul 25 00:44:26 2018 +1000 + + [Kerberos] Add Kerberos authentication support (#32263) + + This commit adds support for Kerberos authentication with a platinum + license. Kerberos authentication support relies on SPNEGO, which is + triggered by challenging clients with a 401 response with the + `WWW-Authenticate: Negotiate` header. A SPNEGO client will then provide + a Kerberos ticket in the `Authorization` header. The tickets are + validated using Java's built-in GSS support. The JVM uses a vm wide + configuration for Kerberos, so there can be only one Kerberos realm. + This is enforced by a bootstrap check that also enforces the existence + of the keytab file. + + In many cases a fallback authentication mechanism is needed when SPNEGO + authentication is not available. In order to support this, the + DefaultAuthenticationFailureHandler now takes a list of failure response + headers. For example, one realm can provide a + `WWW-Authenticate: Negotiate` header as its default and another could + provide `WWW-Authenticate: Basic` to indicate to the client that basic + authentication can be used in place of SPNEGO. + + In order to test Kerberos, unit tests are run against an in-memory KDC + that is backed by an in-memory ldap server. A QA project has also been + added to test against an actual KDC, which is provided by the krb5kdc + fixture. + + Closes #30243 + +commit 99426eb4f81a4a89f2434afc21821093c6c28b1c +Author: David Kyle +Date: Tue Jul 24 15:22:57 2018 +0100 + + [ML] Extract persistent task methods from MlMetadata (#32319) + + Move ML persistent task helper functions to the new class MlTasks + and remove MLMetadataField after moving the string constant to + MlMetadata. + +commit 73a38895fd8e6c38eea726686eb2fce26ee569d0 +Author: Daniel Mitterdorfer +Date: Tue Jul 24 16:17:09 2018 +0200 + + Add Restore Snapshot High Level REST API + + With this commit we add the restore snapshot API to the Java high level + REST client. + + Relates #27205 + Relates #32155 + +commit 59cf600e03f8148434655f2e2e23d88b52c20712 +Author: Christoph Büscher +Date: Tue Jul 24 16:05:43 2018 +0200 + + Register ERR metric with NamedXContentRegistry (#32320) + + This adds the ERR metric to the provided xContent parsers in the module and the + high level rest client registry. Also adding integration tests to make sure the + metric is correctly registered and usable from the client. + +commit a2abc8659595bc51e46550bf32b3c78ca071d664 +Merge: 1a1ae5de1e8 46709f1406a +Author: Colin Goodheart-Smithe +Date: Tue Jul 24 13:13:52 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 46709f1406a01e633d6805b40ed30b79b25ee47a +Author: Vladimir Dolzhenko +Date: Tue Jul 24 14:09:41 2018 +0200 + + fixes broken build for third-party-tests (#32315) + + fixes broken build for third-party-tests (#32315) + Relates #31918 / Closes infra/issues/6085 + +commit a2dbd83db1f5b3ad32c315b48b8229a6269b0438 +Author: Ioannis Kakavas +Date: Tue Jul 24 12:48:14 2018 +0300 + + Allow Integ Tests to run in a FIPS-140 JVM (#31989) + + * Complete changes for running IT in a fips JVM + + - Mute :x-pack:qa:sql:security:ssl:integTest as it + cannot run in FIPS 140 JVM until the SQL CLI supports key/cert. + - Set default JVM keystore/truststore password in top level build + script for all integTest tasks in a FIPS 140 JVM + - Changed top level x-pack build script to use keys and certificates + for trust/key material when spinning up clusters for IT + +commit 177750719db0b274c98ab68dbc719613a8fe98f0 +Author: David Kyle +Date: Tue Jul 24 10:21:34 2018 +0100 + + [DOCS] Rollup Caps API incorrectly mentions GET Jobs API (#32280) + +commit d07b4ecfa358968836aadfce025fcf291dea0ac6 +Author: Andy Bristol +Date: Mon Jul 23 17:20:01 2018 -0700 + + awaitsfix testRandomClusterStateUpdates + + For #32308 + +commit 7c43da280b4689951a7b30e50e0040e3ad41f79a +Author: Zachary Tong +Date: Mon Jul 23 19:50:26 2018 -0400 + + [TEST] add version skip to weighted_avg tests + + Added in 6.4, so we don't want this running on pre-6.4 BWC tests + +commit e2b665c2e6caa5cc78484247f591f437ab99fac5 +Author: Russ Cam +Date: Tue Jul 24 09:21:43 2018 +1000 + + Consistent encoder names (#29492) + + This commit updates encoder names to be consistent within documentation + and align with snake casing convention. + +commit 6ba144ae31a546dbbffc58e0082786a48ec233a7 +Author: Zachary Tong +Date: Mon Jul 23 18:33:15 2018 -0400 + + Add WeightedAvg metric aggregation (#31037) + + Adds a new single-value metrics aggregation that computes the weighted + average of numeric values that are extracted from the aggregated + documents. These values can be extracted from specific numeric + fields in the documents. + + When calculating a regular average, each datapoint has an equal "weight"; it + contributes equally to the final value. In contrast, weighted averages + scale each datapoint differently. The amount that each datapoint contributes + to the final value is extracted from the document, or provided by a script. + + As a formula, a weighted average is the `∑(value * weight) / ∑(weight)` + + A regular average can be thought of as a weighted average where every value has + an implicit weight of `1`. + + Closes #15731 + +commit 55a2d3e0dd6e3f1ac6d37790c8ee5342a0fec764 +Author: Nik Everett +Date: Mon Jul 23 16:50:15 2018 -0400 + + Switch monitoring to new style Requests (#32255) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack/plugin/monitoring` project to use the new + versions. + +commit fe6bb75eb49ad66ca7a6885cc0a80467d44d9545 +Author: Christoph Büscher +Date: Mon Jul 23 22:25:02 2018 +0200 + + Rename ranking evaluation `quality_level` to `metric_score` (#32168) + + The notion of "quality" is an overloaded term in the search ranking evaluation + context. Its usually used to decribe certain levels of "good" vs. "bad" of a + seach result with respect to the users information need. We currently report the + result of the ranking evaluation as `quality_level` which is a bit missleading. + This changes the response parameter name to `metric_score` which fits better. + +commit 1b1aa4ecff2a7d4ac4a37be77226a444259aec95 +Author: Julie Tibshirani +Date: Mon Jul 23 12:25:42 2018 -0700 + + Fix a test bug around nested aggregations and field aliases. (#32287) + + This issue affected both NestedAggregatorTest and ReverseNestedAggregatorTest. + +commit bba1da642d9d8faddcf356af39ffb75f6ad6c145 +Author: James Baiera +Date: Mon Jul 23 15:11:41 2018 -0400 + + Add new permission for JDK11 to load JAAS libraries (#32132) + + Hadoop's security model uses the OS level authentication modules to collect + information about the current user. In JDK 11, the UnixLoginModule makes + use of a new permission to determine if the executing code is allowed to load + the libraries required to pull the user information from the OS. This PR adds + that permission and re-enables the tests that were previously failing when + testing against JDK 11. + +commit 0b72132553a4f7bd29cb6fd6b7fffd860d431874 +Author: Jason Tedor +Date: Mon Jul 23 14:52:23 2018 -0400 + + Silence SSL reload test that fails on JDK 11 + + This commit adds an assumption for an SSL reload test that we are not on + JDK 11 as the test currently fails there. + + Relates #32293 + +commit ac04ba42df10d1747b7ac2054bbea82c47914413 +Author: Andy Bristol +Date: Mon Jul 23 11:45:04 2018 -0700 + + [test] package pre-install java check (#32259) + + This recreates a test that was added to the bats packaging tests + in #31343 but didn't make it over to the java project during when the + linux package tests were ported in #31943 + + When packages are installed but can not locate the java executable, they + should fail with a descriptive message + +commit 1091ec115e59183e9d2f346b9db3c53700b81a3f +Author: Andy Bristol +Date: Mon Jul 23 11:28:24 2018 -0700 + + specify subdirs of lib, bin, modules in package (#32253) + + Explicitly include all subdirectories of these folders in + /usr/share/elasticsearch in package distributions so that they are + managed by the package manager. This change does really have an + effect in the 7.x series, where there are no subdirectories in bin, and + we were already doing this in lib and modules. It does have an effect in + the 6.x series where the bin/x-pack subdirectory was not previously + tracked by the package manager and could be left behind on removal in + rpm distributions. + +commit 1a1ae5de1e8129e0c68855d9ec59f0849366abf5 +Author: Tal Levy +Date: Mon Jul 23 11:14:41 2018 -0700 + + add qa project for running ILM tests against security (#32218) + + This is a bare-bones skeleton for running existing yaml tests with security + enabled. Additional tests which test users and roles should follow + +commit 4804da06bf6475a2b2d37587e60df46b8bf00b69 +Author: Nik Everett +Date: Mon Jul 23 14:07:18 2018 -0400 + + Switch x-pack:core to new style Requests (#32252) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `x-pack:core` project to use the new versions. + +commit d43c3a18b9e729c4d3a9ad3be8efe32eb1722c09 +Author: Andy Bristol +Date: Mon Jul 23 10:57:40 2018 -0700 + + awaitsfix SSLConfigurationReloaderTests + +commit d3c4904fa38e3372e65c15d2e0278367a3c71934 +Author: Jack Conradson +Date: Mon Jul 23 09:12:30 2018 -0700 + + Painless: Clean up add methods in PainlessLookup (#32258) + + This is largely mechanical change that cleans up the addConstructor, addMethod, and + addFields methods in PainlessLookup. Changes include renamed variables, better error + messages, and some minor code movement to make it more maintainable long term. + +commit 33f11e637dc05b5112ffc8c1ca95000a4782fa6d +Author: Andrey Ershov +Date: Mon Jul 23 16:38:55 2018 +0200 + + Fail shard if IndexShard#storeStats runs into an IOException (#32241) + + Fail shard if IndexShard#storeStats runs into an IOException. Closes #29008 + +commit 5f696e15eac1b7c806980f92442ef6df17aa5a9c +Merge: 573a9e6443b 261002283b4 +Author: Colin Goodheart-Smithe +Date: Mon Jul 23 14:49:40 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 261002283b4526a942eb8bdff0a268aaf123734b +Author: Nhat Nguyen +Date: Mon Jul 23 08:56:43 2018 -0400 + + AwaitsFix RecoveryIT#testHistoryUUIDIsGenerated + + Relates #31291 + +commit ff87b7aba42926dea95f65933a7625636d718c87 +Author: Christoph Büscher +Date: Mon Jul 23 11:31:04 2018 +0200 + + Remove unnecessary warning supressions (#32250) + +commit 4b3284f7cbe679b94f10178468566fb0835d8a73 +Author: itsnotv +Date: Mon Jul 23 01:09:52 2018 -0700 + + CCE when re-throwing "shard not available" exception in TransportShardMultiGetAction (#32185) + + ClassCastException can be thrown by callers of TransportActions.isShardNotAvailableException(e) as e is not always an instance of ElasticSearchException + + fixes #32173 + +commit 8f1d15ce61bd9b2c7f4025eaff8d1848cc282a9c +Author: Nicolas Ruflin +Date: Mon Jul 23 07:38:19 2018 +0200 + + Add new fields to monitoring template for Beats state (#32085) + + New data is reported from Beats to the monitoring endpoint. This PR adds the template change necessary for it. See https://github.com/elastic/beats/issues/7521 for more details. + + Queue data is skipped for now as implementation is not finished yet. + +commit 88190299df82df7b61b298c6034be75383eaa71e +Author: Nhat Nguyen +Date: Sun Jul 22 22:14:27 2018 -0400 + + CCR: Fix incorrect read request completion condition (#32266) + + Today we consider a read request is exhausted if from_seqno is equal to + or greater than the max_required_seqno. However, if we stop when + from_seqno equals to the max_required_seqno, we will miss an operation + whose seqno is max_required_seqno because we have not seen that + operation yet. + +commit 8e66a937c9a0afeefe1efb856905ea92cbe9dc37 +Author: Nhat Nguyen +Date: Sun Jul 22 16:05:54 2018 -0400 + + AwaitsFix: forbid completion without contexts test + + Relates #32269 + +commit 0cd25d7b8141d9b8f3c602906be2e106a58f5135 +Merge: b6b596e4713 0a511cc76f7 +Author: Nhat Nguyen +Date: Sat Jul 21 20:00:39 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Improve message when JAVA_HOME not set (#32022) + [TEST] improve REST high-level client naming conventions check (#32244) + +commit 0a511cc76f770f910e512cbbe76ab27f966e1a26 +Author: Alpar Torok +Date: Sat Jul 21 12:50:17 2018 +0000 + + Improve message when JAVA_HOME not set (#32022) + + closes #31399 + +commit b6b596e47136227e96694d65b46250c2def418d4 +Author: Martijn van Groningen +Date: Sat Jul 21 12:38:05 2018 +0200 + + [CCR] Add random shard follow task test (#32188) + + Added shard follow task unit tests that tests whether the shard follow task is able to process randomly generated shard changes api responses. + +commit 1390a849e10568b40a5333059aebfe94f0239411 +Author: Luca Cavanna +Date: Sat Jul 21 10:07:08 2018 +0200 + + [TEST] improve REST high-level client naming conventions check (#32244) + + Check the deprecated methods are effectively deprecated. + Also compare the class rather than their names when checking argument types. + +commit 8ed3624f47a63fd7932e95197ed4025d8b27e98d +Merge: 8e155044434 042424b43ba +Author: Martijn van Groningen +Date: Sat Jul 21 09:06:13 2018 +0200 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: (23 commits) + Switch full-cluster-restart to new style Requests (#32140) + [DOCS] Clarified that you must remove X-Pack plugin when upgrading from pre-6.3. (#32016) + Remove BouncyCastle dependency from runtime (#32193) + INGEST: Extend KV Processor (#31789) (#32232) + INGEST: Make a few Processors callable by Painless (#32170) + Add region ISO code to GeoIP Ingest plugin (#31669) + [Tests] Remove QueryStringQueryBuilderTests#toQuery class assertions (#32236) + Make sure that field aliases count towards the total fields limit. (#32222) + Switch rolling restart to new style Requests (#32147) + muting failing test for internal auto date histogram to avoid failure before fix is merged + MINOR: Remove unused `IndexDynamicSettings` (#32237) + Fix multi level nested sort (#32204) + Enhance Parent circuit breaker error message (#32056) + [ML] Use default request durability for .ml-state index (#32233) + Remove indices stats timeout from monitoring docs + Rename ranking evaluation response section (#32166) + Dependencies: Upgrade to joda time 2.10 (#32160) + Remove aliases resolution limitations when security is enabled (#31952) + Ensure that field aliases cannot be used in multi-fields. (#32219) + TESTS: Check for Netty resource leaks (#31861) + ... + +commit 042424b43ba0e1bf0748e77f9941ccd3ef578f34 +Author: Nik Everett +Date: Fri Jul 20 17:33:15 2018 -0400 + + Switch full-cluster-restart to new style Requests (#32140) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `qa/full-cluster-restart` project to use the new + versions. + + It also fixes a small bug in the test for explain on the `_all` field + that was causing it to not properly invoke `_explain`. + +commit 8e155044434b946608137afef36681ea68c58f1e +Author: Nhat Nguyen +Date: Fri Jul 20 17:18:26 2018 -0400 + + TEST: Fix range issue in ShardChangesActionTests + + We modified the way we calculate to_seqno in #32121 but did not adjust + this test accordingly. If min_seqno equals to max_seqno, the size should be + one instead of zero. + + Relates #32121 + +commit 5525cb1615b7d0a5b4e512305d6e5d2de73f5fcd +Author: debadair +Date: Fri Jul 20 14:17:48 2018 -0700 + + [DOCS] Clarified that you must remove X-Pack plugin when upgrading from pre-6.3. (#32016) + +commit aaa8f842d60f45d6303fb04a9f4fca4ca40dc4ca +Author: Ioannis Kakavas +Date: Sat Jul 21 00:03:58 2018 +0300 + + Remove BouncyCastle dependency from runtime (#32193) + + * Remove BouncyCastle dependency from runtime + + This commit introduces a new gradle project that contains + the classes that have a dependency on BouncyCastle. For + the default distribution, It builds a jar from those and + in puts it in a subdirectory of lib + (/tools/security-cli) along with the BouncyCastle jars. + This directory is then passed in the + ES_ADDITIONAL_CLASSPATH_DIRECTORIES of the CLI tools + that use these classes. + + BouncyCastle is removed as a runtime dependency (remains + as a compileOnly one) from x-pack core and x-pack security. + +commit 7aa8a0a92787b116edd9044c131115e68459d932 +Author: Armin Braun +Date: Fri Jul 20 22:32:50 2018 +0200 + + INGEST: Extend KV Processor (#31789) (#32232) + + * INGEST: Extend KV Processor (#31789) + + Added more capabilities supported by LS to the KV processor: + * Stripping of brackets and quotes from values (`include_brackets` in corresponding LS filter) + * Adding key prefixes + * Trimming specified chars from keys and values + + Refactored the way the filter is configured to avoid conditionals during execution. + Refactored Tests a little to not have to add more redundant getters for new parameters. + + Relates #31786 + * Add documentation + +commit e21692e3870a9e4437f9884628bfb90ca19fe3dd +Author: Armin Braun +Date: Fri Jul 20 21:10:35 2018 +0200 + + INGEST: Make a few Processors callable by Painless (#32170) + + * INGEST: Make a few Processors callable by Painless + * Extracted a few stateless String processors as well as the json processor to static methods and whitelisted them in Painless + * provide whitelist from processors plugin + +commit ac6340865560edd221e634b541312d4ed3999b7e +Author: Nick Peihl +Date: Fri Jul 20 11:23:29 2018 -0700 + + Add region ISO code to GeoIP Ingest plugin (#31669) + +commit 54d896c4ede28500a01e308be7c0f504a1835b63 +Author: Christoph Büscher +Date: Fri Jul 20 19:08:59 2018 +0200 + + [Tests] Remove QueryStringQueryBuilderTests#toQuery class assertions (#32236) + + Currently we check that the queries that QueryStringQueryBuilder#toQuery returns + is one out of a list of many Lucene query classes. This list has extended a lot over time, + since QueryStringQueryBuilder can build all sort of queries. This makes the test hard to + maintain. The recent addition of alias fields which build a BlendedTermQuery show how + easy this test breaks. Also the current assertions doesn't add a lot in terms of catching + errors. This is why we decided to remove this check. + + Closes #32234 + +commit af0c1d30fe8570d33cd413b6a81e737ecd754524 +Author: Julie Tibshirani +Date: Fri Jul 20 10:06:07 2018 -0700 + + Make sure that field aliases count towards the total fields limit. (#32222) + +commit c6c9075ca49af7a1aebdcbb01aa36c5adcc4a55f +Author: Nik Everett +Date: Fri Jul 20 12:01:50 2018 -0400 + + Switch rolling restart to new style Requests (#32147) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `qa/rolling-upgrade` project to use the new + versions. + +commit 320f1d263f51f7b6c36158b6774a05844dcab86e +Author: Paul Sanwald +Date: Fri Jul 20 11:20:51 2018 -0400 + + muting failing test for internal auto date histogram to avoid failure before fix is merged + +commit 91a0daf0e49f21e5dc909ad395b3eb8fad4e9075 +Author: Armin Braun +Date: Fri Jul 20 17:14:17 2018 +0200 + + MINOR: Remove unused `IndexDynamicSettings` (#32237) + +commit d2092100199a5eaffbd24ba6ebd46b4d8bff3cb2 +Author: David Turner +Date: Fri Jul 20 16:05:44 2018 +0100 + + Introduce deterministic task queue (#32197) + + The cluster coordination layer relies on timeouts to ensure that a cluster can + successfully form, and must also deal with concurrent activity in the cluster. + This commit introduces some infrastructure that will help us to + deterministically test components that use concurrency and/or timeouts. + +commit 6ed1ad0b6ffd5eba5be0807d7ea96aefe722840f +Author: Jim Ferenczi +Date: Fri Jul 20 16:55:11 2018 +0200 + + Fix multi level nested sort (#32204) + + The parent filter for nested sort should always match **all** parents regardless + of the child queries. It is used to find the boundaries of a single parent and we use + the child query to match all the filters set in the nested tree so there is no need to + repeat the nested filters. + With this change we ensure that we build bitset filters + only to find the root docs (or the docs at the level where the sort applies) that can be reused + among queries. + + Closes #31554 + Closes #32130 + Closes #31783 + + Co-authored-by: Dominic Bevacqua + +commit 74aa7b0815c048271a5536aad7941ca72a0d8f6c +Author: Lee Hinman +Date: Fri Jul 20 08:52:45 2018 -0600 + + Enhance Parent circuit breaker error message (#32056) + + * Enhance Parent circuit breaker error message + + This adds information about either the current real usage (if tracking "real" + memory usage) or the child breaker usages to the exception message when the + parent circuit breaker trips. + + The messages now look like: + + ``` + [parent] Data too large, data for [my_request] would be [211288064/201.5mb], which is larger than the limit of [209715200/200mb], usages [request=157286400/150mb, fielddata=54001664/51.5mb, in_flight_requests=0/0b, accounting=0/0b] + ``` + + Or when tracking real memory usage: + + ``` + [parent] Data too large, data for [request] would be [251/251b], which is larger than the limit of [200/200b], real usage: [181/181b], new bytes reserved: [70/70b] + ``` + + * Only call currentMemoryUsage once by returning structured object + +commit ac960bfa6b6bd96e83cd186bf95667f48be1e0e6 +Author: David Kyle +Date: Fri Jul 20 15:49:37 2018 +0100 + + [ML] Use default request durability for .ml-state index (#32233) + + The initial decision to use async durability was made a long time ago + for performance reasons. That argument no longer applies and we + prefer the safety of request durability. + +commit fe574f89f856a58df9467a83d9171e32407cfbb8 +Author: Nhat Nguyen +Date: Fri Jul 20 08:39:32 2018 -0400 + + CCR: Translog op on primary should have versionType + + Normally translog operations will not be replayed on the primary. + Following engine is an exception where we replay translog on both + primary and replica as a non-primary strategy. Even though we won't use + the version_type in the following engine, we still need to pass a valid + value for the primary operation in order not to trip assertions in an + engine. + + This commit passes version_type EXTERNAL for translog operation if its + origin is primary. + + Relates #31945 + +commit a6b7497fdc1340f277c5a82c3a402db1c149ee04 +Author: Martijn van Groningen +Date: Fri Jul 20 14:12:05 2018 +0200 + + [CCR] Add more unit tests for shard follow task (#32121) + + The added tests are based on specific scenarios as described in the test plan. + Before this change the ShardFollowNodeTaskTests contained more random like tests, + but these have been removed and in a followup pr better random tests will + be added in a new test class as is described in the test plan. + +commit 6afb661ca5393853df38d9a104568e1ea8b34d0a +Author: Daniel Mitterdorfer +Date: Fri Jul 20 12:55:20 2018 +0200 + + Remove indices stats timeout from monitoring docs + + With this commit we remove the documentation for the setting + `xpack.monitoring.collection.indices.stats.timeout` which has already + been removed in code. + + Closes #32133 + Relates #32229 + +commit 5cbd9ad177e4284a2fef799d01920bd09648de70 +Author: Christoph Büscher +Date: Fri Jul 20 11:43:46 2018 +0200 + + Rename ranking evaluation response section (#32166) + + Currently the ranking evaluation response contains a 'unknown_docs' section + for each search use case in the evaluation set. It contains document ids for + results in the search hits that currently don't have a quality rating. + This change renames it to `unrated_docs`, which better reflects its purpose. + +commit 384cc5455b8bb26c086d7b0afe33ef825d40b188 +Author: Yannick Welsch +Date: Fri Jul 20 11:17:42 2018 +0200 + + Add core coordination algorithm for cluster state publishing (#32171) + + Adds the core coordination algorithm that guarantees safety for cluster state publishing. This is a + straight-forward port of the transition rules from the formal model at + https://github.com/elastic/elasticsearch-formal-models/blob/master/ZenWithTerms/tla/ZenWithTerms.tla + +commit c5cde96691df84353f1f796db40c9df0efc6825e +Author: Alexander Reelsen +Date: Fri Jul 20 10:18:38 2018 +0200 + + Dependencies: Upgrade to joda time 2.10 (#32160) + + Changelog: http://www.joda.org/joda-time/changes-report.html + +commit 00a6ad0e9ee874bc211a2fc55824b0c14975c7a5 +Author: Luca Cavanna +Date: Fri Jul 20 09:23:32 2018 +0200 + + Remove aliases resolution limitations when security is enabled (#31952) + + Resolving wildcards in aliases expression is challenging as we may end + up with no aliases to replace the original expression with, but if we + replace with an empty array that means _all which is quite the opposite. + Now that we support and serialize the original requested aliases, + whenever aliases are replaced we will be able to know what was + initially requested. `MetaData#findAliases` can then be updated to not + return anything in case it gets empty aliases, but the original aliases + were not empty. That means that empty aliases are interpreted as _all + only if they were originally requested that way. + + Relates to #31516 + +commit 0f0068b91cad02fe504e2ecc1a23742275120758 +Author: Julie Tibshirani +Date: Fri Jul 20 00:18:54 2018 -0700 + + Ensure that field aliases cannot be used in multi-fields. (#32219) + +commit 24068a773dcd5725ff72c1a55f3ab4bd4a0df576 +Author: Armin Braun +Date: Fri Jul 20 09:12:32 2018 +0200 + + TESTS: Check for Netty resource leaks (#31861) + + * Enabled advanced leak detection when loading `EsTestCase` + * Added custom `Appender` to collect leak logs and check for logged errors in a way similar to what is done for the `StatusLogger` + * Fixes #20398 + +commit 573a9e6443b49c453919e17cfdcf20d41f8e6312 +Merge: 1acdd379689 a9a9598e02a +Author: Colin Goodheart-Smithe +Date: Fri Jul 20 07:56:57 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit a9a9598e02ac6edb27bd4940196a2e27a78d7288 +Author: Alpar Torok +Date: Fri Jul 20 05:20:35 2018 +0000 + + Require Gradle 4.9 as minimum version (#32200) + + Do the check in the build plugin as well to be more informative to users + of build-tools. + + We use APIs specific to Gradle 4.9 so earlier will not work. + +commit d0f3ed5abdfba8745c6d391b0559915cd45328c9 +Merge: d88c76e02b2 c7a41c501ac +Author: Nhat Nguyen +Date: Thu Jul 19 23:02:45 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Painless: Simplify Naming in Lookup Package (#32177) + Handle missing values in painless (#32207) + add support for write index resolution when creating/updating documents (#31520) + ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864) + Remove indication of future multi-homing support (#32187) + Rest test - allow for snapshots to take 0 milliseconds + Make x-pack-core generate a pom file + Rest HL client: Add put watch action (#32026) + Build: Remove pom generation for plugin zip files (#32180) + Fix comments causing errors with Java 11 + Fix rollup on date fields that don't support epoch_millis (#31890) + Detect and prevent configuration that triggers a Gradle bug (#31912) + [test] port linux package packaging tests (#31943) + Revert "Introduce a Hashing Processor (#31087)" (#32178) + Remove empty @return from JavaDoc + Adjust SSLDriver behavior for JDK11 changes (#32145) + [test] use randomized runner in packaging tests (#32109) + Add support for field aliases. (#32172) + Painless: Fix caching bug and clean up addPainlessClass. (#32142) + Call setReferences() on custom referring tokenfilters in _analyze (#32157) + Fix BwC Tests looking for UUID Pre 6.4 (#32158) + Improve docs for search preferences (#32159) + use before instead of onOrBefore + Add more contexts to painless execute api (#30511) + Add EC2 credential test for repository-s3 (#31918) + A replica can be promoted and started in one cluster state update (#32042) + Fix Java 11 javadoc compile problem + Fix CP for namingConventions when gradle home has spaces (#31914) + Fix `range` queries on `_type` field for singe type indices (#31756) + [DOCS] Update TLS on Docker for 6.3 (#32114) + ESIndexLevelReplicationTestCase doesn't support replicated failures but it's good to know what they are + Remove versionType from translog (#31945) + Switch distribution to new style Requests (#30595) + Build: Skip jar tests if jar disabled + Painless: Add PainlessClassBuilder (#32141) + Build: Make additional test deps of check (#32015) + Disable C2 from using AVX-512 on JDK 10 (#32138) + Build: Move shadow customizations into common code (#32014) + Painless: Fix Bug with Duplicate PainlessClasses (#32110) + Remove empty @param from Javadoc + Re-disable packaging tests on suse boxes + Docs: Fix missing example script quote (#32010) + [ML] Wait for aliases in multi-node tests (#32086) + [ML] Move analyzer dependencies out of categorization config (#32123) + Ensure to release translog snapshot in primary-replica resync (#32045) + Handle TokenizerFactory TODOs (#32063) + Relax TermVectors API to work with textual fields other than TextFieldType (#31915) + Updates the build to gradle 4.9 (#32087) + Mute :qa:mixed-cluster indices.stats/10_index/Index - all’ + Check that client methods match API defined in the REST spec (#31825) + Enable testing in FIPS140 JVM (#31666) + Fix put mappings java API documentation (#31955) + Add exclusion option to `keep_types` token filter (#32012) + [Test] Modify assert statement for ssl handshake (#32072) + +commit 6f2b7dc9fe6143d2cc2bf26d83983a3a49c40b90 +Author: Tim Vernum +Date: Fri Jul 20 12:51:52 2018 +1000 + + Fix AD / vagrant based tests for #30953 + + These tests were creating a SSL service that was not aware of the + realm that they were trying to test. + This no longer works. + +commit c32981db6bc4b04383a801ba40c6781d22f262fe +Author: Tim Vernum +Date: Fri Jul 20 10:09:28 2018 +1000 + + Detect old trial licenses and mimic behaviour (#32209) + + Prior to 6.3 a trial license default to security enabled. Since 6.3 + they default to security disabled. If a cluster is upgraded from <6.3 + to >6.3, then we detect this and mimic the old behaviour with respect + to security. + +commit c7a41c501ac0c8ef6a6cd96c2abc7c4fbd7fb76e +Author: Jack Conradson +Date: Thu Jul 19 16:35:03 2018 -0700 + + Painless: Simplify Naming in Lookup Package (#32177) + + This removes some extraneous naming syntax and makes clear the meaning of certain + naming conventions without ambiguities (stricter) within the lookup package. Purely + mechanical change. Note this does not cover a large portion of the + PainlessLookupBuilder and PainlessLookup yet as there are several more follow up PRs for these incoming. + +commit 4c68dfe0016673bbdb1885384d1934858180d521 +Author: Mayya Sharipova +Date: Thu Jul 19 17:41:06 2018 -0400 + + Handle missing values in painless (#32207) + + Throw an exception for doc['field'].value + if this document is missing a value for the field. + + After deprecation changes have been backported to 6.x, + make this a default behaviour in 7.0 + + Closes #29286 + +commit 1acdd379689fd2470501306e4e3e351a8218e84b +Merge: 4ae0d4e802d 9ae6905657b +Author: Tal Levy +Date: Thu Jul 19 12:33:10 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 9ae6905657b310ede5d968d71f4f3265eb26db5b +Author: Tal Levy +Date: Thu Jul 19 09:17:49 2018 -0700 + + add support for write index resolution when creating/updating documents (#31520) + + Now write operations like Index, Delete, Update rely on the write-index associated with + an alias to operate against. This means writes will be accepted even when an alias points to multiple indices, so long as one is the write index. Routing values will be used from the AliasMetaData for the alias in the write-index. All read operations are left untouched. + +commit 4ae0d4e802d5339bce6073428f04034c61382024 +Author: Tal Levy +Date: Thu Jul 19 07:02:37 2018 -0700 + + add ILM ShrinkAction IT (#32176) + + - adds integration test for shrink action + - expands phases used by allocate/replicas to include `cold` + +commit 7c0fc209bf78e4824ca1f232b84a1dab22bc2dfa +Author: Vladimir Dolzhenko +Date: Thu Jul 19 12:54:38 2018 +0200 + + ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864) + + ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864) + + Closes #26913 + +commit f6d7854f76216247ed70ecf2dabb4d6edd978fff +Author: David Turner +Date: Thu Jul 19 11:33:46 2018 +0100 + + Remove indication of future multi-homing support (#32187) + + We do not support intra-cluster connections on multiple interfaces, but the + documentation indicates that we will in future. In fact there is currently no + plan to support this, so the forward-looking documentation is misleading. This + commit + + - removes the misleading sentence + - fixes that a transport profile affects outbound connections, not inbound ones + - tidies up some nearby text + +commit f7da55aa3c878a845795ece9ae21327801a4204c +Author: Boaz Leskes +Date: Thu Jul 19 10:44:32 2018 +0200 + + Rest test - allow for snapshots to take 0 milliseconds + + Sometimes we get lucky and things are fast :) + +commit 9aa562d3dd943103e6ae5dbfb2305778e572c76a +Author: Ryan Ernst +Date: Thu Jul 19 01:53:08 2018 -0700 + + Make x-pack-core generate a pom file + + This was a forgotten part of #32180 + +commit 202894b8322571c447bc20a6262fd49da89f9bf5 +Author: Alexander Reelsen +Date: Thu Jul 19 10:40:54 2018 +0200 + + Rest HL client: Add put watch action (#32026) + + Relates #29827 + + This implementation behaves like the current transport client, that you basically cannot configure a Watch POJO representation as an argument to the put watch API, but only a bytes reference. You can use the the `WatchSourceBuilder` from the `org.elasticsearch.plugin:x-pack-core` dependency to build watches. + + This commit also changes the license type to trial, so that watcher is available in high level rest client tests. + + /cc @hub-cap + +commit 185689ad75955522872acfc7cf21170036680fea +Author: Ryan Ernst +Date: Thu Jul 19 01:24:49 2018 -0700 + + Build: Remove pom generation for plugin zip files (#32180) + + In 1.x and 2.x, plugins were published to maven and the plugin + installer downloaded them from there. This was later changed to install + from the download service, and in 5.0 plugin zips were no longer + published to maven. However, the build still currently produces an + unused pom file. This is troublesome in the special case when the main + jar of a plugin needs to be published (and thus needs a pom file of + the same name). + + closes #31946 + +commit f232c36c197d7d2e470823e672f66e85e4e1e20c +Author: Christoph Büscher +Date: Thu Jul 19 09:41:32 2018 +0200 + + Fix comments causing errors with Java 11 + +commit 644a92f1582ceeda6981c0653a5e455f5ec43e3c +Author: Jim Ferenczi +Date: Thu Jul 19 09:34:23 2018 +0200 + + Fix rollup on date fields that don't support epoch_millis (#31890) + + The rollup indexer uses a range query to select the next page + of results based on the last time bucket of the previous round + and the `delay` configured on the rollup job. This query uses + the `epoch_millis` format implicitly but doesn't set the `format`. + This result in errors during the rollup job if the field + definition doesn't allow this format. It can also miss documents + if the format is not accepted but another format in the field + definition is able to parse the query (e.g.: `epoch_second`). + This change ensures that we use `epoch_millis` as the only format + to parse the rollup range query. + +commit 38e2e1d553280de0f49ed9bb5a2c7df4187d4ed5 +Author: Alpar Torok +Date: Thu Jul 19 06:46:58 2018 +0000 + + Detect and prevent configuration that triggers a Gradle bug (#31912) + + * Detect and prevent configuration that triggers a Gradle bug + + As we found in #31862, this can lead to a lot of wasted time as it's not + immediatly obvius what's going on. + Givent how many projects we have it's getting increasingly easier to run + into gradle/gradle#847. + +commit aae0133847254ce38dd2dc3ee01a10b486589243 +Author: Andy Bristol +Date: Wed Jul 18 17:18:00 2018 -0700 + + [test] port linux package packaging tests (#31943) + + Add packaging tests for the linux package distributions to the java test + project and remove them from bats. Most of the tests that lived in + 30_deb_package.bats and 40_rpm_package.bats are applicable to both + package types and are combined into a single type of test case. Others + are separated out into separate cases to make their intent more clear + + For #26741 + +commit 27d663b64b775355bb787355888132837f28db2e +Author: Tal Levy +Date: Wed Jul 18 16:44:05 2018 -0700 + + Revert "Introduce a Hashing Processor (#31087)" (#32178) + + This reverts commit 8c78fe711486a3f722879d8f7cf89632daf793e0. + +commit 63749498ea92dc03ac424f7f5764eed752eb567e +Author: Michael Basnight +Date: Wed Jul 18 13:28:25 2018 -0500 + + Remove empty @return from JavaDoc + +commit a019e723b5475f0ca77063dab45fbcccd1d6559a +Merge: 6c77449a877 90fcb384481 +Author: Tal Levy +Date: Wed Jul 18 11:10:12 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 90fcb38448158d3a5d595307ee9f5c3d64b42e7f +Author: Tim Brooks +Date: Wed Jul 18 11:49:42 2018 -0600 + + Adjust SSLDriver behavior for JDK11 changes (#32145) + + This is related to #32122. A number of things changed related to adding + TLS 1.3 support in JDK11. Some exception messages and other SSLEngine + behavior changed. This commit fixes assertions on exception messages. + Additionally it identifies two bugs related to how the SSLDriver behaves + in regards to JDK11 changes. Finally, it mutes a tests until correct + behavior can be identified. There is another open issue for that muted + test (#32144). + +commit e20f59aa716f094296960afdc39f927a987481b5 +Author: Andy Bristol +Date: Wed Jul 18 10:26:26 2018 -0700 + + [test] use randomized runner in packaging tests (#32109) + + Use the randomized runner from the test framework and add some basic + logging to make the packaging tests behave more similarly to how we use + junit in the rest of the project + +commit 15ff3da653c5dd9eae7f7a654a9ee6a931d478b7 +Author: Julie Tibshirani +Date: Wed Jul 18 09:33:09 2018 -0700 + + Add support for field aliases. (#32172) + + * Add basic support for field aliases in index mappings. (#31287) + * Allow for aliases when fetching stored fields. (#31411) + * Add tests around accessing field aliases in scripts. (#31417) + * Add documentation around field aliases. (#31538) + * Add validation for field alias mappings. (#31518) + * Return both concrete fields and aliases in DocumentFieldMappers#getMapper. (#31671) + * Make sure that field-level security is enforced when using field aliases. (#31807) + * Add more comprehensive tests for field aliases in queries + aggregations. (#31565) + * Remove the deprecated method DocumentFieldMappers#getFieldMapper. (#32148) + +commit 605dc49c4828fe74d039e361cbe283bbf19a9a49 +Author: Jack Conradson +Date: Wed Jul 18 09:29:52 2018 -0700 + + Painless: Fix caching bug and clean up addPainlessClass. (#32142) + + This change cleans up the addPainlessClass methods by doing the following things: + + * Rename many variable names to match the new conventions described in the JavaDocs + for PainlessLookup + * Decouples Whitelist.Class from adding a PainlessClass directly + * Adds a second version of addPainlessClass that is intended for use to add future + defaults in a follow PR + + This change also fixes the method and field caches by storing Classes instead of Strings + since it would technically be possible now that the whitelists are extendable to have + different Classes with the same name. It was convenient to add this change together + since some of the new constants are shared. + + Note the changes are largely mechanical again where all the code behavior should + remain the same. + +commit cfb30144c9175d1f1538af0aa5c1841f72159128 +Author: Alan Woodward +Date: Wed Jul 18 14:43:20 2018 +0100 + + Call setReferences() on custom referring tokenfilters in _analyze (#32157) + + When building custom tokenfilters without an index in the _analyze endpoint, + we need to ensure that referring filters are correctly built by calling + their #setReferences() method + + Fixes #32154 + +commit 6de1f96cadd8d187e3732b710eb10264f733d422 +Author: Armin Braun +Date: Wed Jul 18 15:32:36 2018 +0200 + + Fix BwC Tests looking for UUID Pre 6.4 (#32158) + + * UUID field was added for #31791 and only went into 6.4 and 7.0 + * Fixes #32119 + +commit 380b45b96590a313507f26ff2b0e3a2af54f4706 +Author: David Turner +Date: Wed Jul 18 12:58:17 2018 +0100 + + Improve docs for search preferences (#32159) + + Today it is unclear what guarantees are offered by the search preference + feature, and we claim a guarantee that is stronger than what we really offer: + + > A custom value will be used to guarantee that the same shards will be used + > for the same custom value. + + This commit clarifies this documentation. + + Forward-port of #32098 to `master`. + +commit 53ab470264261b25e799696085c134df09ddaf77 +Author: Martijn van Groningen +Date: Wed Jul 18 13:33:57 2018 +0200 + + use before instead of onOrBefore + +commit 1924f5d07c9f739821bc8c749a5314a074dab36b +Author: Martijn van Groningen +Date: Wed Jul 18 12:42:07 2018 +0200 + + Add more contexts to painless execute api (#30511) + + This change adds two contexts the execute scripts against: + + * SEARCH_SCRIPT: Allows to run scripts in a search script context. + This context is used in `function_score` query's script function, + script fields, script sorting and `terms_set` query. + + * FILTER_SCRIPT: Allows to run scripts in a filter script context. + This context is used in the `script` query. + + In both contexts a index name needs to be specified and a sample document. + The document is needed to create an in-memory index that the script can + access via the `doc[...]` and other notations. The index name is needed + because a mapping is needed to index the document. + + Examples: + + ``` + POST /_scripts/painless/_execute + { + "script": { + "source": "doc['field'].value.length()" + }, + "context" : { + "search_script": { + "document": { + "field": "four" + }, + "index": "my-index" + } + } + } + ``` + + Returns: + + ``` + { + "result": 4 + } + ``` + + POST /_scripts/painless/_execute + { + "script": { + "source": "doc['field'].value.length() <= params.max_length", + "params": { + "max_length": 4 + } + }, + "context" : { + "filter_script": { + "document": { + "field": "four" + }, + "index": "my-index" + } + } + } + + Returns: + + ``` + { + "result": true + } + ``` + + Also changed PainlessExecuteAction.TransportAction to use TransportSingleShardAction + instead of HandledAction, because now in case score or filter contexts are used + the request needs to be redirected to a node that has an active IndexService + for the index being referenced (a node with a shard copy for that index). + +commit 6c77449a87764147099b1e35c0ec191de3202d7a +Merge: 5d3a53843a2 8235b254abf +Author: Colin Goodheart-Smithe +Date: Wed Jul 18 11:41:23 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 8235b254abf274e99b7af91d125edc0a9dbe627a +Author: Vladimir Dolzhenko +Date: Wed Jul 18 12:18:00 2018 +0200 + + Add EC2 credential test for repository-s3 (#31918) + + Add EC2 credential test for repository-s3 + + Relates to #26913 + +commit 5856c396ddd9e36094c5471d436be100bb586e60 +Author: Boaz Leskes +Date: Wed Jul 18 11:30:44 2018 +0200 + + A replica can be promoted and started in one cluster state update (#32042) + + When a replica is fully recovered (i.e., in `POST_RECOVERY` state) we send a request to the master + to start the shard. The master changes the state of the replica and publishes a cluster state to that + effect. In certain cases, that cluster state can be processed on the node hosting the replica + *together* with a cluster state that promotes that, now started, replica to a primary. This can + happen due to cluster state batched processing or if the master died after having committed the + cluster state that starts the shard but before publishing it to the node with the replica. If the master + also held the primary shard, the new master node will remove the primary (as it failed) and will also + immediately promote the replica (thinking it is started). + + Sadly our code in IndexShard didn't allow for this which caused [assertions](https://github.com/elastic/elasticsearch/blob/13917162ad5c59a96ccb4d6a81a5044546c45c22/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java#L482) to be tripped in some of our tests runs. + +commit ef5e8d8d8a82afcc9d9f49a4c8a868d559ffef93 +Author: Christoph Büscher +Date: Wed Jul 18 10:34:42 2018 +0200 + + Fix Java 11 javadoc compile problem + + Java 11 complains with a "type arguments not allowed here" error when types are + used in javadoc links it seems. Simply removing it. + +commit 69cbdc47eb97544dfde51a042860fca110c6a20c +Author: Alpar Torok +Date: Wed Jul 18 08:23:59 2018 +0000 + + Fix CP for namingConventions when gradle home has spaces (#31914) + + * Fix CP for namingConventions when gradle home has spaces + + Closes #31736. + + Probably not Windows specific, just not common to have spaces on Linux. + +commit 15f95a9f934d7ff4a4c3ab9e70a8877cf046e9ad +Author: Christoph Büscher +Date: Wed Jul 18 09:12:28 2018 +0200 + + Fix `range` queries on `_type` field for singe type indices (#31756) + + With the introduction of single types in 6.x, the `_type` field is no longer + indexed, which leads to certain queries that were working before throw errors + now. One such query is the `range` query, that, if performed on a single typer + index, currently throws an IAE since the field is not indexed. + This change adds special treatment for this case in the TypeFieldMapper, + comparing the range queries lower and upper bound to the one existing type and + either returns a MatchAllDocs or a MatchNoDocs query. + + Relates to #31632 + Closes #31476 + +commit 487cfc3b3f9df22f46e112461429578732844903 +Author: Toby McLaughlin +Date: Wed Jul 18 17:07:31 2018 +1000 + + [DOCS] Update TLS on Docker for 6.3 (#32114) + + Remove references to the `platinum` image and add a self-generated trial + licence to the example for TLS on Docker. + + Fixes elastic/elasticsearch-docker#176 + +commit 93d7468f3a49fba26a4176476b8f219e86ecdef0 +Author: Boaz Leskes +Date: Tue Jul 17 18:06:16 2018 +0200 + + ESIndexLevelReplicationTestCase doesn't support replicated failures but it's good to know what they are + + Sometimes we have a test failure that hits an `UnsupportedOperationException` in this infrastructure. When + debugging you want to know what caused this unexpected failure, but right now we're silent about it. This + commit adds some information to the `UnsupportedOperationException` + + Relates to #32127 + +commit df1380b8d3267d70771a640c7bac5a04fb63a8e1 +Author: Nhat Nguyen +Date: Tue Jul 17 21:59:48 2018 -0400 + + Remove versionType from translog (#31945) + + With the introduction of sequence number, we no longer use versionType to + resolve out of order collision in replication and recovery requests. + + This PR removes removes the versionType from translog. We can only remove + it in 7.0 because it is still required in a mixed cluster between 6.x and 5.x. + +commit 351bbb89063dc00dbea76f8e09300ede9d1be28c +Author: Nik Everett +Date: Tue Jul 17 20:25:27 2018 -0400 + + Switch distribution to new style Requests (#30595) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `distribution/archives/integ-test-zip` project + to use the new versions. + +commit 91d8371325bfd4a6cbf88d4e77f4769f1d8bdea0 +Author: Nik Everett +Date: Tue Jul 17 18:41:31 2018 -0400 + + Build: Skip jar tests if jar disabled + + The shadow plugin disables the jar task but we still attempted to extract + the jar to see if it had the right license and notice file. This skips + the extraction and those tests if the jar is built for any reason which + fixes projects that use the shadow plugin. + +commit 03c16cd0e33a3c869c2a0347147771b895ba20e8 +Author: Jack Conradson +Date: Tue Jul 17 13:54:49 2018 -0700 + + Painless: Add PainlessClassBuilder (#32141) + + Several pieces of data in PainlessClass cannot be passed in at the time the + PainlessClass is created so it must be "frozen" after all the data is collected. This means + PainlessClass is currently serving two functions as both a builder and a set of data. This + separates the two pieces into clearly distinct values. + + This change also removes the PainlessMethodKey in favor of a simple String. The goal is + to have the painless method key be completely internal to the PainlessLookup eventually + and this simplifies the way there. Note that this was added since PainlessClass and + PainlessClassBuilder were already being changed instead of a follow up PR. + +commit 6371d51866e6b015f378fad568abbc0a7df94e2d +Author: Ryan Ernst +Date: Tue Jul 17 13:14:46 2018 -0700 + + Build: Make additional test deps of check (#32015) + + This commit moves additional unit test runners from being dependencies + of the test task to dependencies of check. Without this change, + reproduce lines are incorrect due to the additional test runner not + matching any of the reproduce class/method info. + + closes #31964 + +commit d170ab33150ff96ed14cc47ea61e44136cc2dc1f +Author: Jason Tedor +Date: Tue Jul 17 15:49:20 2018 -0400 + + Disable C2 from using AVX-512 on JDK 10 (#32138) + + The C2 compiler in JDK 10 appears to have an issue compiling to AVX-512 + instructions (on hardware that supports such). As a workaround, this + commit adds a JVM flag on JDK 10+ to disable the use of AVX-512 + instructions until a fix is introduced to the JDK. Instead, we use a + flag to enable AVX and AVX2 only. + + Note: Based on my reading of the C2 code, this flag does not appear to + have any impact on hardware that does not support AVX2. I have tested + this manually on an Intel Atom C2538 processor that supports neither AVX + nor AVX2. I have also tested this manually on an Intel i5-3317U + processor that supports AVX but not AVX2. + +commit 1b97652a4cc1aa2acbe761464745abb35dd8faa0 +Author: Nik Everett +Date: Tue Jul 17 14:20:41 2018 -0400 + + Build: Move shadow customizations into common code (#32014) + + Moves the customizations to the build to produce nice shadow jars and + javadocs into common build code, mostly BuildPlugin with a little into + the root build.gradle file. This means that any project that applies the + shadow plugin will automatically be set up just like the high level rest + client: + * The non-shadow jar will not be built + * The shadow jar will not have a "classifier" + * Tests will run against the shadow jar + * Javadoc will include all of the shadowed classes + * Service files in `META-INF/services` will be merged + +commit 1c63eb108164af85a4006650f9c9753f64eb5622 +Author: Jack Conradson +Date: Tue Jul 17 10:33:38 2018 -0700 + + Painless: Fix Bug with Duplicate PainlessClasses (#32110) + + When building the PainlessMethods and PainlessFields they stored a reference to a + PainlessClass. This reference was prior to "freezing" the PainlessClass so the data was + both incomplete and mutable. This has been replaced with a target java class instead + since the PainlessClass is accessible through a java class now and it requires no special + modifications to get around a chicken and egg issue. + +commit 9cdbd1bd85e92181ee258b9c874be985fcc61de5 +Author: Ioannis Kakavas +Date: Tue Jul 17 20:25:38 2018 +0300 + + Remove empty @param from Javadoc + +commit ad78f7394291775083fe88761b24c5653f9d77df +Author: Yannick Welsch +Date: Tue Jul 17 18:28:00 2018 +0200 + + Add term and config to cluster state (#32100) + + Adds the publication term and the last accepted and committed configurations to the cluster state, + following the formal model in + https://github.com/elastic/elasticsearch-formal-models/blob/master/ZenWithTerms/tla/ZenWithTerms.tla + The term represents the reign of a master, and the last committed / accepted configurations + represent the set of quorums that cluster state changes will require (If there's no reconfiguration, + last accepted and last committed configurations coincide). + +commit 0f7d2044c321d49c2e1ed71273d4f39fc7cfeb9a +Author: Andy Bristol +Date: Tue Jul 17 09:15:11 2018 -0700 + + Re-disable packaging tests on suse boxes + + This reverts commit 14d7e2c7b2d3761a361edd720f98353f856936a4. + +commit efb4e97cfb5dc35da640d8e2b8d249b11db54c1e +Author: aptxx +Date: Tue Jul 17 23:42:24 2018 +0800 + + Docs: Fix missing example script quote (#32010) + +commit d88c76e02b28928003ec7fff439ec73741901703 +Author: Martijn van Groningen +Date: Tue Jul 17 17:39:49 2018 +0200 + + [CCR] Initial replication group based tests (#32024) + + Tests shard follow task in the context of a leader and follower ReplicationGroup, + in order to test how the shard follow logic reacts to certain shard related + failure scenarios. + + More tests will need to be added, but this indicates what changes need to be made + to have these tests. + + Relates to #30102 + +commit 2bd40a79276f1e73ebfdda8c0c28a26800b4e349 +Author: David Kyle +Date: Tue Jul 17 16:21:53 2018 +0100 + + [ML] Wait for aliases in multi-node tests (#32086) + +commit 99c2a82c04872fe09e98aa461e744718b91ac3f1 +Author: David Roberts +Date: Tue Jul 17 15:01:12 2018 +0100 + + [ML] Move analyzer dependencies out of categorization config (#32123) + + The ML config classes will shortly be moved to the X-Pack protocol + library to allow the ML APIs to be moved to the high level REST + client. Dependencies on server functionality should be removed + from the config classes before this is done. + + This change is entirely about moving code between packages. It + does not add or remove any functionality or tests. + +commit ef81c1df57587d6e44dd750ad36926a5c8bce894 +Author: Nhat Nguyen +Date: Tue Jul 17 09:41:34 2018 -0400 + + Ensure to release translog snapshot in primary-replica resync (#32045) + + Previously we create a translog snapshot inside the resync method, + and that snapshot will be closed by the resync listener. However, if + the resync method throws an exception before the resync listener + is initialized, the translog snapshot won't be released. + + Closes #32030 + +commit ed3b44fb4cd6f957facf6d0f38daa08eeecd52ea +Author: Armin Braun +Date: Tue Jul 17 14:14:02 2018 +0200 + + Handle TokenizerFactory TODOs (#32063) + + * Don't replace Replace TokenizerFactory with Supplier, this approach was rejected in #32063 + * Remove unused parameter from constructor + +commit a7e477126f5b9c18decf24314cacd2ad5456b2d5 +Author: markharwood +Date: Tue Jul 17 13:11:10 2018 +0100 + + Relax TermVectors API to work with textual fields other than TextFieldType (#31915) + + This changes the field-eligibility test to check one level up in the class hierarchy to allow any subclasses of StringFieldType. + Closes #31902 + +commit b43fe560a43ee5c43bae5105b07aae811a69b1ef +Author: Alpar Torok +Date: Tue Jul 17 11:41:31 2018 +0000 + + Updates the build to gradle 4.9 (#32087) + + There are fixes to the dependency report, most importantly for us, + it still works even if `failOnVersionConflict` would fail the build. + +commit b655c11dbee65d5331819f04a15ace80b639cfbd +Author: David Kyle +Date: Tue Jul 17 10:34:45 2018 +0100 + + Mute :qa:mixed-cluster indices.stats/10_index/Index - all’ + +commit 049966a829918a61cf3ac181e709183c9ef686e8 +Author: Luca Cavanna +Date: Tue Jul 17 11:26:28 2018 +0200 + + Check that client methods match API defined in the REST spec (#31825) + + We have been encountering name mismatches between API defined in our + REST spec and method names that have been added to the high-level REST + client. We should check this automatically to prevent furher mismatches, + and correct all the current ones. + + This commit adds a test for this and corrects the issues found by it. + +commit 006c79a80d265484294ea8edf544732f8f26dbe7 +Author: Martijn van Groningen +Date: Tue Jul 17 10:25:51 2018 +0200 + + [CCR] Improve retry mechanism when making remote calls from shard follow task (#31930) + + Closes #31816 + +commit 9e529d9d584ae6f6978ac09056b73d9ccdde4a3c +Author: Ioannis Kakavas +Date: Tue Jul 17 10:54:10 2018 +0300 + + Enable testing in FIPS140 JVM (#31666) + + Ensure our tests can run in a FIPS JVM + + JKS keystores cannot be used in a FIPS JVM as attempting to use one + in order to init a KeyManagerFactory or a TrustManagerFactory is not + allowed.( JKS keystore algorithms for private key encryption are not + FIPS 140 approved) + This commit replaces JKS keystores in our tests with the + corresponding PEM encoded key and certificates both for key and trust + configurations. + Whenever it's not possible to refactor the test, i.e. when we are + testing that we can load a JKS keystore, etc. we attempt to + mute the test when we are running in FIPS 140 JVM. Testing for the + JVM is naive and is based on the name of the security provider as + we would control the testing infrastrtucture and so this would be + reliable enough. + Other cases of tests being muted are the ones that involve custom + TrustStoreManagers or KeyStoreManagers, null TLS Ciphers and the + SAMLAuthneticator class as we cannot sign XML documents in the + way we were doing. SAMLAuthenticator tests in a FIPS JVM can be + reenabled with precomputed and signed SAML messages at a later stage. + + IT will be covered in a subsequent PR + +commit 36165265ce6010a33147a6e3c5296a64f989944d +Author: Christoph Büscher +Date: Tue Jul 17 09:09:03 2018 +0200 + + Fix put mappings java API documentation (#31955) + + The current docs of the put-mapping Java API is currently broken. It its current + form, it creates an index and uses the whole mapping definition given as a JSON + string as the type name. Since we didn't check the index created in the + IndicesDocumentationIT so far this went unnoticed. + + This change adds test to catch this error to the documentation test, changes the + documentation so it works correctly now and adds an input validation to + PutMappingRequest#buildFromSimplifiedDef() which was used internally to reject + calls where no mapping definition is given. + + Closes #31906 + +commit 61486680a25fc16299f666385970fd7dc310271c +Author: Christoph Büscher +Date: Tue Jul 17 09:04:41 2018 +0200 + + Add exclusion option to `keep_types` token filter (#32012) + + Currently the `keep_types` token filter includes all token types specified using + its `types` parameter. Lucenes TypeTokenFilter also provides a second mode where + instead of keeping the specified tokens (include) they are filtered out + (exclude). This change exposes this option as a new `mode` parameter that can + either take the values `include` (the default, if not specified) or `exclude`. + + Closes #29277 + +commit 6717df3c2deb35ce367abedd556e337d3df7c41f +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Tue Jul 17 15:43:03 2018 +1000 + + [Test] Modify assert statement for ssl handshake (#32072) + + There have been changes in error messages for `SSLHandshakeException`. + This has caused a couple of failures in our tests. + This commit modifies test verification to assert on exception type of + class `SSLHandshakeException`. + There was another issue in Java11 which caused NPE. The bug has now + been fixed on Java11 - early access build 22. + Bug Ref: https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8206355 + Enable the skipped tests due to this bug. + + Closes #31940 + +commit d0c9cf26a9d0e9b9ccb9368a3aed755366fb6c15 +Merge: f5e21682609 4b5071f2d08 +Author: Martijn van Groningen +Date: Tue Jul 17 07:36:34 2018 +0200 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: + Add Index UUID to `/_stats` Response (#31871) + Painless: Move and Rename Several Methods in the lookup package (#32105) + Bypass highlight query terms extraction on empty fields (#32090) + Switch non-x-pack to new style requests (#32106) + [Rollup] Add new capabilities endpoint for concrete rollup indices (#30401) + Revert "[test] disable packaging tests for suse boxes" + SQL: allow LEFT and RIGHT as function names (#32066) + DOCS: put LIMIT 10 to the SQL query (#32065) + [test] turn on host io cache for opensuse (#32053) + Tweaked Elasticsearch Service links for SEO + +commit 4b5071f2d08a2ba23eb6d84d47192453c6e68b11 +Author: Armin Braun +Date: Tue Jul 17 06:50:21 2018 +0200 + + Add Index UUID to `/_stats` Response (#31871) + + * Add "uuid" field to each index's section in the `/_stats` response + * closes #31791 + +commit 15740d622971551e6595017be21dc6dc58a6ea84 +Author: Jack Conradson +Date: Mon Jul 16 16:13:48 2018 -0700 + + Painless: Move and Rename Several Methods in the lookup package (#32105) + +commit f699cb9f5565bfe6825ae196969a45b4e6365d31 +Author: Jim Ferenczi +Date: Tue Jul 17 00:26:01 2018 +0200 + + Bypass highlight query terms extraction on empty fields (#32090) + + Dealing with empty fields in the highlight phase can + slow down the query because the query terms extraction is done independently + on each field. This change shortcuts the highlighting performed by the unified highlighter + for fields that are not present in the document. In such cases there is nothing to higlight so + we don't need to visit the query to build the highligh builder. + +commit d596447f3dc186855af1e32dc06a75d06feea89c +Author: Nik Everett +Date: Mon Jul 16 17:44:19 2018 -0400 + + Switch non-x-pack to new style requests (#32106) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes most of the calls not in X-Pack to their new versions. + +commit 791b9b147cb1763678500ca91cc28e7fa171f40d +Author: Zachary Tong +Date: Mon Jul 16 17:20:50 2018 -0400 + + [Rollup] Add new capabilities endpoint for concrete rollup indices (#30401) + + This introduces a new GetRollupIndexCaps API which allows the user to retrieve rollup capabilities of a specific rollup index (or index pattern). This is distinct from the existing RollupCaps endpoint. + + - Multiple jobs can be stored in multiple indices and point to a single target data index pattern (logstash-*). The existing API finds capabilities/config of all jobs matching that data index pattern. + - One rollup index can hold data from multiple jobs, targeting multiple data index patterns. This new API finds the capabilities based on the concrete rollup indices. + +commit 14d7e2c7b2d3761a361edd720f98353f856936a4 +Author: Andy Bristol +Date: Mon Jul 16 13:57:17 2018 -0700 + + Revert "[test] disable packaging tests for suse boxes" + + This reverts commit 30d6fd3ca74c44de27f8314fe7c070ac1d23819a. + +commit 20ea72e2f26ed0c4d21b6865820298a26091daa8 +Author: Costin Leau +Date: Mon Jul 16 23:42:44 2018 +0300 + + SQL: allow LEFT and RIGHT as function names (#32066) + + Due to the way ANTLR works, any declared tokens need to be accounted for + manually inside function names (otherwise a different rule gets applied). + + Fix #32046 + +commit 5b65e2ccbf66370524b66a11305ebd4555cad27b +Author: ahmedakef +Date: Mon Jul 16 22:00:57 2018 +0200 + + DOCS: put LIMIT 10 to the SQL query (#32065) + + Provides a more precise equivalent SQL query for the aggregation example in the getting started guide. + +commit 69c85331b0f574323994dc1ccb4f70ab0e122a4c +Author: Andy Bristol +Date: Mon Jul 16 13:02:53 2018 -0700 + + [test] turn on host io cache for opensuse (#32053) + + The hope is that this will resolve the problems with very slow io we're + seeing on this box in #30295 + +commit ff8260c234a6f522941528e82a11fcde6f181ce8 +Author: debadair +Date: Mon Jul 16 12:58:08 2018 -0700 + + Tweaked Elasticsearch Service links for SEO + +commit f5e21682609731d1b611df2b1e6959a198292829 +Merge: 89a590a59a8 cccc3f7a641 +Author: Martijn van Groningen +Date: Mon Jul 16 21:49:57 2018 +0200 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: (21 commits) + Tweaked Elasticsearch Service links for SEO + Watcher: Store username on watch execution (#31873) + Use correct formatting for links (#29460) + Painless: Separate PainlessLookup into PainlessLookup and PainlessLookupBuilder (#32054) + Scripting: Remove dead code from painless module (#32064) + [Rollup] Replace RollupIT with a ESRestTestCase version (#31977) + [TEST] Consistent algorithm usage (#32077) + [Rollup] Fix duplicate field names in test (#32075) + Ensure only parent breaker trips in unit test + Unmute field collapsing rest tests + Fix BWC check after backport + [Tests] Fix failure due to changes exception message (#32036) + Remove unused params from SSource and Walker (#31935) + [Test] Mute MlJobIT#testDeleteJobAfterMissingAliases + Turn off real-mem breaker in REST tests + Turn off real-mem breaker in single node tests + Fix broken OpenLDAP Vagrant QA test + Cleanup Duplication in `PainlessScriptEngine` (#31991) + SCRIPTING: Remove unused MultiSearchTemplateRequestBuilder (#32049) + Fix compile issues introduced by merge (#32058) + ... + +commit cccc3f7a641f4e9cbe73adf968ffe1419c715a3f +Author: debadair +Date: Mon Jul 16 12:47:57 2018 -0700 + + Tweaked Elasticsearch Service links for SEO + +commit 637cac90614018d6e46c781547ddac2000d71459 +Author: Michael Basnight +Date: Mon Jul 16 14:20:16 2018 -0500 + + Watcher: Store username on watch execution (#31873) + + There is currently no way to see what user executed a watch. This commit + adds the decrypted username to each execution in the watch history, in a + new field "user". + + Closes #31772 + +commit ecd05d5be408d5b864189c1c51b9154d627e536b +Author: DeDe Morton +Date: Mon Jul 16 12:11:24 2018 -0700 + + Use correct formatting for links (#29460) + +commit 2a1a28f19c63a268330b3fe351dda731ee0f6627 +Author: Jack Conradson +Date: Mon Jul 16 11:15:29 2018 -0700 + + Painless: Separate PainlessLookup into PainlessLookup and PainlessLookupBuilder (#32054) + +commit b1479bbed889787ee9febb3f6b3d3414ba071754 +Author: Armin Braun +Date: Mon Jul 16 18:43:00 2018 +0200 + + Scripting: Remove dead code from painless module (#32064) + +commit 59191b4998afb7dd7093e44d14f06f4f3e7c4740 +Author: Zachary Tong +Date: Mon Jul 16 10:47:46 2018 -0400 + + [Rollup] Replace RollupIT with a ESRestTestCase version (#31977) + + The old RollupIT was a node IT, an flaky for a number of reasons. + This new version is an ESRestTestCase and should be a little more robust. + + This was added to the multi-node QA tests as that seemed like the most + appropriate location. It didn't seem necessary to create a whole new + QA module. + + Note: The only test that was ported was the "Big" test for validating + a larger dataset. The rest of the tests are represented in existing + yaml tests. + + Closes #31258 + Closes #30232 + Related to #30290 + +commit ef7ccd1c074b7fb9a2d1e388d517e6d20c131903 +Author: Ioannis Kakavas +Date: Mon Jul 16 16:41:56 2018 +0300 + + [TEST] Consistent algorithm usage (#32077) + + Ensure that the same algorithm is used for settings and + change password requests for consistency, even if we + do not expext to reach the code where the algorithm is + checked for now. + Completes a7eaa409e804f218aa06fd02d9166b9a5998b48a + +commit a3b608d616fe8f9b8bb10bb60a97a9f39ec53a6c +Author: Jim Ferenczi +Date: Mon Jul 16 15:25:45 2018 +0200 + + [Rollup] Fix duplicate field names in test (#32075) + + This commit ensures that random field names do not clash with the explicit field names + set by the tests. + + Closes #32067 + +commit 1fef139c11065aee746a18f4d76ca65eb369ece3 +Author: Daniel Mitterdorfer +Date: Mon Jul 16 13:50:17 2018 +0200 + + Ensure only parent breaker trips in unit test + + With this commit we raise the limit of the child circuit breaker used in + the unit test for the circuit breaker service so it is high enough to trip + only the parent circuit breaker. The previous limit was 300 bytes but + theoretically (considering overhead) we could reach 346 bytes. Thus any + value larger than 300 bytes could trip the child circuit breaker leading + to spurious failures. + + Relates #31767 + +commit 44f0c1df39ebc34e9613e3e5bcad78a3308a9068 +Author: Jim Ferenczi +Date: Mon Jul 16 12:03:28 2018 +0200 + + Unmute field collapsing rest tests + + BWC tests can run now that master and 6x branch are aligned. + Closes #32055 + +commit fa59bb10999b20d487ff12a890ee8cfa0f414826 +Author: Jim Ferenczi +Date: Mon Jul 16 11:59:59 2018 +0200 + + Fix BWC check after backport + + Relates #31808 + +commit 3587d8872e9d2b43c6b66f7d6ec7aafe6a7c1df0 +Author: Christoph Büscher +Date: Mon Jul 16 11:22:42 2018 +0200 + + [Tests] Fix failure due to changes exception message (#32036) + + Java 11 seems to get more verbose on the ClassCastException we check for in + SearchDocumentationIT. This changes the test from asserting the exact exception + message to only checking the two classes involved are part of the message. + + Closes #32029 + +commit ca4c4f736ae1d73d503e3e5b530e025117d15e94 +Author: Christoph Büscher +Date: Mon Jul 16 10:54:23 2018 +0200 + + Remove unused params from SSource and Walker (#31935) + + The "source" field in SSource seems unused. If removed, it can also be removed + from the ctor, which in turn makes is possible to delete the sourceText in the + Walker class. + +commit a14db2f9d6df79d5b38befc2657a689ce6d90cbf +Author: Jim Ferenczi +Date: Mon Jul 16 10:53:51 2018 +0200 + + [Test] Mute MlJobIT#testDeleteJobAfterMissingAliases + + Relates #32034 + +commit e0cfa1689c17a660e0e3e957262e642ac760a7d8 +Author: Daniel Mitterdorfer +Date: Mon Jul 16 10:44:04 2018 +0200 + + Turn off real-mem breaker in REST tests + + With this commit we disable the real-memory circuit breaker in REST + tests as this breaker is based on real memory usage over which we have + no (full) control in tests and the REST client is not yet ready to retry + on circuit breaker exceptions. + + This is only meant as a temporary measure to avoid spurious test + failures while we ensure that the REST client can handle those + situations appropriately. + + Closes #32050 + Relates #31767 + Relates #31986 + Relates #32074 + +commit 016e8760f0b44e816bae541a822d5d2fbb8f3021 +Author: Daniel Mitterdorfer +Date: Mon Jul 16 10:40:36 2018 +0200 + + Turn off real-mem breaker in single node tests + + With this commit we disable the real-memory circuit breaker in tests + that inherit from `ESSingleNodeTestCase`. As this breaker is based on + real memory usage over which we have no (full) control in tests and + their purpose is also not to test the circuit breaker, we use the + deterministic circuit breaker implementation that only accounts for + explicitly reserved memory. + + Closes #32047 + Relates #32071 + +commit 5d3a53843a21098de3d1a342482db555eb660976 +Merge: ebf0dd45d26 edbea73f24b +Author: Colin Goodheart-Smithe +Date: Mon Jul 16 09:30:07 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit edbea73f24b8f85b05a4f6af1a8d1c7b3b63796f +Author: Tim Vernum +Date: Mon Jul 16 15:43:41 2018 +1000 + + Fix broken OpenLDAP Vagrant QA test + + This was broken due to c662565 but the problem didn't get detected as + CI builds typically don't run vagrant tests + +commit b65c586cef0c8b46aaab26bc3ea9ef81c7653aa9 +Author: Armin Braun +Date: Sat Jul 14 13:37:59 2018 +0200 + + Cleanup Duplication in `PainlessScriptEngine` (#31991) + + * Cleanup Duplication in `PainlessScriptEngine` + * Extract duplicate building of compiler settings to method + * Remove dead method params + dead constant in `ScriptProcessor` + +commit ccf61264101437c1d188c6dec24fc9348dfe0a63 +Author: Armin Braun +Date: Sat Jul 14 09:03:35 2018 +0200 + + SCRIPTING: Remove unused MultiSearchTemplateRequestBuilder (#32049) + + * Ever since 46e8d97813addd8c57fa54d2c700d26a171f2dbb this class is unused + +commit a612404b1f84ce92a24a69b9ad0b538f4c8f3428 +Author: Tim Brooks +Date: Fri Jul 13 23:37:15 2018 -0600 + + Fix compile issues introduced by merge (#32058) + + The build was broken due to some issues with the merging of #32018. A + method that was public went private before the PR was merged. That did + not cause a merge conflict (so the PR was merged successfully). But it + did cause the build to fail. + +commit 305bfea9c3c174d35136fddbf2bcd8544b4ea3bd +Author: Tim Brooks +Date: Fri Jul 13 16:41:02 2018 -0600 + + Add nio http transport to security plugin (#32018) + + This is related to #27260. It adds the SecurityNioHttpServerTransport + to the security plugin. It randomly uses the nio http transport in + security integration tests. + +commit 89a590a59a850b624f3ca418f12789b116b834f2 +Merge: e26f3e0c261 3679d00a746 +Author: Nhat Nguyen +Date: Fri Jul 13 18:08:34 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Replace Ingest ScriptContext with Custom Interface (#32003) + Mute failing tests + +commit 3679d00a7465c9d8602e991be76390bd905fddd7 +Author: Armin Braun +Date: Fri Jul 13 23:26:10 2018 +0200 + + Replace Ingest ScriptContext with Custom Interface (#32003) + + * Replace Ingest ScriptContext with Custom Interface + * Make org.elasticsearch.ingest.common.ScriptProcessorTests#testScripting more precise + * Don't mock script factory in ScriptProcessorTests + * Adjust mock script plugin in IT for new API + +commit 466235288e8fc74d265fbf6f98d044572a60a460 +Author: Christoph Büscher +Date: Fri Jul 13 23:08:18 2018 +0200 + + Mute failing tests + + Relates to #32055 + +commit e26f3e0c2617ff6107a28ce54e0164bfd6398e9f +Merge: 08c2f551c00 867c49600c6 +Author: Nhat Nguyen +Date: Fri Jul 13 16:55:57 2018 -0400 + + Merge branch 'master' into ccr + +commit 08c2f551c005778974aa1655a38f1f38beee4371 +Author: Nhat Nguyen +Date: Fri Jul 13 16:39:58 2018 -0400 + + Ensure LuceneChangesSnapshot close once + +commit 867c49600c69e7e7baf6559bc1c2b736d6405c3f +Author: Zachary Tong +Date: Fri Jul 13 16:10:39 2018 -0400 + + [Rollup] Histo group config should support scaled_floats (#32048) + + Metric config already whitelist scaled_floats, but it wasn't added to + the histo group config. This centralizes the mapping types map + so that both metrics and histo (and any future configs) use the same + map. + + Fixes #32035 + +commit 42ca520377098edcbd851efa640259b6a81371a5 +Author: Jack Conradson +Date: Fri Jul 13 13:07:26 2018 -0700 + + Clean Up Snapshot Create Rest API (#31779) + + Make SnapshotInfo and CreateSnapshotResponse parsers lenient for backwards compatibility. Remove extraneous fields from CreateSnapshotRequest toXContent. + +commit feb07559aadaa3175d8d7d5401c1a55438ee6451 +Author: Paul Sanwald +Date: Fri Jul 13 14:59:11 2018 -0400 + + fix typo + +commit 1f72afa77314d27208e710398f65f701c865ea1f +Author: Michael Basnight +Date: Fri Jul 13 13:49:47 2018 -0500 + + Watcher: Make settings reloadable (#31746) + + This commit allows for rebuilding watcher secure secrets via the + reload_secure_settings API call. The commit also renames a method in the + Notification Service to make it a bit more readable. + +commit b1bf643e41b2bf2bc1360887b4768836c9159214 +Author: Vladimir Dolzhenko +Date: Fri Jul 13 20:05:49 2018 +0200 + + lazy snapshot repository initialization (#31606) + + lazy snapshot repository initialization + +commit 0edb096eb4763e2149c6c67aa8842198ecefb108 +Author: Colin Goodheart-Smithe +Date: Fri Jul 13 18:08:35 2018 +0100 + + Adds a new auto-interval date histogram (#28993) + + * Adds a new auto-interval date histogram + + This change adds a new type of histogram aggregation called `auto_date_histogram` where you can specify the target number of buckets you require and it will find an appropriate interval for the returned buckets. The aggregation works by first collecting documents in buckets at second interval, when it has created more than the target number of buckets it merges these buckets into minute interval bucket and continues collecting until it reaches the target number of buckets again. It will keep merging buckets when it exceeds the target until either collection is finished or the highest interval (currently years) is reached. A similar process happens at reduce time. + + This aggregation intentionally does not support min_doc_count, offest and extended_bounds to keep the already complex logic from becoming more complex. The aggregation accepts sub-aggregations but will always operate in `breadth_first` mode deferring the computation of sub-aggregations until the final buckets from the shard are known. min_doc_count is effectively hard-coded to zero meaning that we will insert empty buckets where necessary. + + Closes #9572 + + * Adds documentation + + * Added sub aggregator test + + * Fixes failing docs test + + * Brings branch up to date with master changes + + * trying to get tests to pass again + + * Fixes multiBucketConsumer accounting + + * Collects more buckets than needed on shards + + This gives us more options at reduce time in terms of how we do the + final merge of the buckeets to produce the final result + + * Revert "Collects more buckets than needed on shards" + + This reverts commit 993c782d117892af9a3c86a51921cdee630a3ac5. + + * Adds ability to merge within a rounding + + * Fixes nonn-timezone doc test failure + + * Fix time zone tests + + * iterates on tests + + * Adds test case and documentation changes + + Added some notes in the documentation about the intervals that can bbe + returned. + + Also added a test case that utilises the merging of conseecutive buckets + + * Fixes performance bug + + The bug meant that getAppropriate rounding look a huge amount of time + if the range of the data was large but also sparsely populated. In + these situations the rounding would be very low so iterating through + the rounding values from the min key to the max keey look a long time + (~120 seconds in one test). + + The solution is to add a rough estimate first which chooses the + rounding based just on the long values of the min and max keeys alone + but selects the rounding one lower than the one it thinks is + appropriate so the accurate method can choose the final rounding taking + into account the fact that intervals are not always fixed length. + + Thee commit also adds more tests + + * Changes to only do complex reduction on final reduce + + * merge latest with master + + * correct tests and add a new test case for 10k buckets + + * refactor to perform bucket number check in innerBuild + + * correctly derive bucket setting, update tests to increase bucket threshold + + * fix checkstyle + + * address code review comments + + * add documentation for default buckets + + * fix typo + +commit 2c3ea43f45a6364b215acca428f29a7af7f66d10 +Author: Ryan Ernst +Date: Fri Jul 13 09:33:27 2018 -0700 + + HLRC: Add xpack usage api (#31975) + + This commit adds the _xpack/usage api to the high level rest client. + Currently in the transport api, the usage data is exposed in a limited + fashion, at most giving one level of helper methods for the inner keys + of data, but then exposing thos subobjects as maps of objects. Rather + than making parsers for every set of usage data from each feature, this + PR exposes the entire set of usage data as a map of maps. + +commit bf7689071b4e580cc3056ac231c0329fabd7f01d +Author: Michael Basnight +Date: Fri Jul 13 11:13:10 2018 -0500 + + Add secure setting for watcher email password (#31620) + + Other watcher actions already account for secure settings in their + sensitive settings, whereas the email sending action did not. This adds + the ability to optionally set a secure_password for email accounts. + +commit c1a81e552fa014c5654c42bcda002eae7d10236c +Author: Michael Basnight +Date: Fri Jul 13 11:12:03 2018 -0500 + + Watcher: cleanup ensureWatchExists use (#31926) + + Previously, the ensureWatchExists was overridable. This commit makes + it final so that it cannot be overridden, and cleans up some redundant + code in the process. + +commit 82cdb574cf746ef2c4336181a8399b8bcb11ff3e +Author: Christoph Büscher +Date: Fri Jul 13 17:41:28 2018 +0200 + + Mute ML AutodetectMemoryLimitIT#testTooManyPartitions on Windows (#32044) + + Adding assumption to not run this test on Windows temporarily. + + Relates to #32033 + +commit 80492cacfcadbb0ea513914e1012a725f8739b55 +Author: Mayya Sharipova +Date: Fri Jul 13 11:40:03 2018 -0400 + + Add second level of field collapsing (#31808) + + * Put second level collapse under inner_hits + + Closes #24855 + +commit f9791cf158ad6403b2872897d45ff7c773269569 +Author: Alan Woodward +Date: Fri Jul 13 15:49:26 2018 +0100 + + Remove deprecated AnalysisPlugin#requriesAnalysisSettings method (#32037) + +commit 1bf205f862a32640d4febcd61cea49a04e406fa0 +Author: Michael Basnight +Date: Fri Jul 13 09:33:29 2018 -0500 + + Test: Fix a second case of bad watch creation + + There was still a case with a null text that allowed for 0 attachments + to be created. This commit ensures that greater than zero are created + if the text is null. Otherwise, it uses the same logic to create 0 to 3 + random attachments. + + Closes #31948 + +commit b7f07f03edb9c5be2665d65eafc8fe9ad95d0fa1 +Author: Zachary Tong +Date: Fri Jul 13 10:07:42 2018 -0400 + + [Rollup] Use composite's missing_bucket (#31402) + + We can leverage the composite agg's new `missing_bucket` feature on + terms groupings. This means the aggregation criteria used in the indexer + will now return null buckets for missing keys. + + Because all buckets are now returned (even if a key is null), + we can guarantee correct doc counts with + "combined" jobs (where a job rolls up multiple schemas). This was + previously impossible since composite would ignore documents that + didn't have _all_ the keys, meaning non-overlapping schemas would + cause composite to return no buckets. + + Note: date_histo does not use `missing_bucket`, since a timestamp is + always required. + + The docs have been adjusted to recommend a single, combined job. It + also makes reference to the previous issue to help users that are upgrading + (rather than just deleting the sections). + +commit bc1284eb28a68cc2b5d18a3181ac72ff12804348 +Author: Clinton Gormley +Date: Fri Jul 13 15:48:14 2018 +0200 + + Docs: Restyled cloud link in getting started + +commit 9a928756e979e6f2f1469aa00b89c2c050788d68 +Author: Clinton Gormley +Date: Fri Jul 13 15:40:38 2018 +0200 + + Docs: Change formatting of Cloud options + +commit 92a9bb2b965fb401543aedf71ea7707da55e1a15 +Author: Christoph Büscher +Date: Fri Jul 13 14:34:02 2018 +0200 + + Re-instate link in StringFunctionUtils javadocs + + The previous errors in compileJava were not cause by the brackets but my the + content of the @link section. Corrected this so its a working javadoc link again. + +commit a01e26a39baf0908cd30351c7f17e6a19d00276b +Author: Alan Woodward +Date: Fri Jul 13 13:13:21 2018 +0100 + + Correct spelling of AnalysisPlugin#requriesAnalysisSettings (#32025) + + Because this is a static method on a public API, and one that we encourage + plugin authors to use, the method with the typo is deprecated in 6.x + rather than just renamed. + +commit e31a877a6462f21e26ae53007d77f1be9e85f034 +Author: Christoph Büscher +Date: Fri Jul 13 11:06:01 2018 +0200 + + Fix problematic chars in javadoc + + Java 11 complains about unescaped ">" characters in javadocs. Also fixed some + compiler complaints about javadoc in StringFunctionUtils. + +commit 69c2b2d8027a2c3c07d33316184db07c611df2e9 +Author: Christoph Büscher +Date: Fri Jul 13 10:56:15 2018 +0200 + + [Tests] Mute failing SearchDocumentationIT#testSearchRequestAggregations + +commit 23198ea5512891c86e422c69e45970e357c9fade +Author: Alpar Torok +Date: Fri Jul 13 08:35:12 2018 +0000 + + Convert Version to Java - clusterformation part1 (#32009) + + Implement buildSrc Version in java + + - This allows to move all all .java files from .groovy. + - Will prevent eclipse from tangling up in this setup + - make it possible to use Version from Java + +commit f174f72fee0c38ab078641b097e4745c7899eb46 +Author: Daniel Mitterdorfer +Date: Fri Jul 13 10:08:28 2018 +0200 + + Circuit-break based on real memory usage + + With this commit we introduce a new circuit-breaking strategy to the parent + circuit breaker. Contrary to the current implementation which only accounts for + memory reserved via child circuit breakers, the new strategy measures real heap + memory usage at the time of reservation. This allows us to be much more + aggressive with the circuit breaker limit so we bump it to 95% by default. The + new strategy is turned on by default and can be controlled with the new cluster + setting `indices.breaker.total.userealmemory`. + + Note that we turn it off for all integration tests with an internal test cluster + because it leads to spurious test failures which are of no value (we cannot + fully control heap memory usage in tests). All REST tests, however, will make + use of the real memory circuit breaker. + + Relates #31767 + +commit d2461643cdfc147d78a2c78164bd96033e918340 +Author: David Roberts +Date: Fri Jul 13 08:57:33 2018 +0100 + + [ML] Move open job failure explanation out of root cause (#31925) + + When an ML job cannot be allocated to a node the exception + contained an explanation of why the job couldn't be + allocated to each node in the cluster. For large clusters + this was not particularly easy to read and made the error + displayed in the UI look very scary. + + This commit changes the structure of the error to an outer + ElasticsearchException with a high level message and an + inner IllegalStateException containing the detailed + explanation. Because the definition of root cause is the + innermost ElasticsearchException the detailed explanation + will not be the root cause (which is what Kibana displays). + + Fixes #29950 + +commit c662565f291db3c7e95f9cfa75e764999cd1bbfb +Author: Tim Vernum +Date: Fri Jul 13 16:40:09 2018 +1000 + + Access SSL contexts using names instead of Settings (#30953) + + Historically we have loaded SSL objects (such as SSLContext, + SSLIOSessionStrategy) by passing in the SSL settings, constructing a + new SSL configuration from those settings and then looking for a + cached object that matches those settings. + + The primary issue with this approach is that it requires a fully + configured Settings object to be available any time the SSL context + needs to be loaded. If the Settings include SecureSettings (such as + passwords for keys or keystores) then this is not true, and the cached + SSL object cannot be loaded at runtime. + + This commit introduces an alternative approach of naming every cached + ssl configuration, so that it is possible to load the SSL context for + a named configuration (such as "xpack.http.ssl"). This means that the + calling code does not need to have ongoing access to the secure + settings that were used to load the configuration. + + This change also allows monitoring exporters to use SSL passwords + from secure settings, however an exporter that uses a secure SSL setting + (e.g. truststore.secure_password) may not have its SSL settings updated + dynamically (this is prevented by a settings validator). + Exporters without secure settings can continue to be defined and updated + dynamically. + +commit b1f5d361b3af58d75ccc161f700e4416a6f6a5c2 +Merge: 4d180175c67 332c134517b +Author: Nhat Nguyen +Date: Thu Jul 12 17:33:57 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [TEST] Mute SlackMessageTests.testTemplateRender + Docs: Explain closing the high level client + [ML] Re-enable memory limit integration tests (#31328) + [test] disable packaging tests for suse boxes + Add nio transport to security plugin (#31942) + XContentTests : Insert random fields at random positions (#30867) + Force execution of fetch tasks (#31974) + Fix unreachable error condition in AmazonS3Fixture (#32005) + Tests: Fix SearchFieldsIT.testDocValueFields (#31995) + Add Expected Reciprocal Rank metric (#31891) + [ML] Get ForecastRequestStats doc in RestoreModelSnapshotIT (#31973) + SQL: Add support for single parameter text manipulating functions (#31874) + [ML] Ensure immutability of MlMetadata (#31957) + Tests: Mute SearchFieldsIT.testDocValueFields() + muted tests due to #31940 + Work around reported problem in eclipse (#31960) + Move build integration tests out of :buildSrc project (#31961) + Tests: Remove use of joda time in some tests (#31922) + [Test] Reactive 3rd party tests on CI (#31919) + SQL: Support for escape sequences (#31884) + SQL: HAVING clause should accept only aggregates (#31872) + Docs: fix typo in datehistogram (#31972) + Switch url repository rest tests to new style requests (#31944) + Switch reindex tests to new style requests (#31941) + Docs: Added note about cloud service to installation and getting started + [DOCS] Removes alternative docker pull example (#31934) + Add Snapshots Status API to High Level Rest Client (#31515) + ingest: date_index_name processor template resolution (#31841) + Test: fix null failure in watcher test (#31968) + Switch test framework to new style requests (#31939) + Switch low level rest tests to new style Requests (#31938) + Switch high level rest tests to new style requests (#31937) + [ML] Mute test failing due to Java 11 date time format parsing bug (#31899) + [TEST] Mute SlackMessageTests.testTemplateRender + Fix assertIngestDocument wrongfully passing (#31913) + Remove unused reference to filePermissionsCache (#31923) + rolling upgrade should use a replica to prevent relocations while running a scroll + HLREST: Bundle the x-pack protocol project (#31904) + Increase logging level for testStressMaybeFlush + Added lenient flag for synonym token filter (#31484) + [X-Pack] Beats centralized management: security role + licensing (#30520) + HLRest: Move xPackInfo() to xPack().info() (#31905) + Docs: add security delete role to api call table (#31907) + [test] port archive distribution packaging tests (#31314) + Watcher: Slack message empty text (#31596) + [ML] Mute failing DetectionRulesIT.testCondition() test + Fix broken NaN check in MovingFunctions#stdDev() (#31888) + Date: Add DateFormatters class that uses java.time (#31856) + [ML] Switch native QA tests to a 3 node cluster (#31757) + Change trappy float comparison (#31889) + Fix building AD URL from domain name (#31849) + Add opaque_id to audit logging (#31878) + re-enable backcompat tests + add support for is_write_index in put-alias body parsing (#31674) + Improve release notes script (#31833) + [DOCS] Fix broken link in painless example + Handle missing values in painless (#30975) + Remove the ability to index or query context suggestions without context (#31007) + Ingest: Enable Templated Fieldnames in Rename (#31690) + [Docs] Fix typo in the Rollup API Quick Reference (#31855) + Ingest: Add ignore_missing option to RemoveProc (#31693) + Add template config for Beat state to X-Pack Monitoring (#31809) + Watcher: Add ssl.trust email account setting (#31684) + Remove link to oss-MSI (#31844) + Painless: Restructure Definition/Whitelist (#31879) + HLREST: Add x-pack-info API (#31870) + +commit 332c134517bdcf11acd74c205482245c692186fb +Author: Christoph Büscher +Date: Thu Jul 12 22:41:35 2018 +0200 + + [TEST] Mute SlackMessageTests.testTemplateRender + +commit 9822a6f911cba6e00a8828432e2094e289935435 +Author: Nik Everett +Date: Thu Jul 12 16:22:42 2018 -0400 + + Docs: Explain closing the high level client + + It looks like we weren't clear on when and why you should close the high + level client and folks were closing it after every request which is not + efficient. This explains why you should close the client and when so + this shouldn't be as common. + + Closes #32001 + +commit de2eb85aaf1448ebeac680de830db8c504048a00 +Author: Tom Veasey +Date: Thu Jul 12 20:56:05 2018 +0100 + + [ML] Re-enable memory limit integration tests (#31328) + +commit 30d6fd3ca74c44de27f8314fe7c070ac1d23819a +Author: Andy Bristol +Date: Thu Jul 12 11:12:37 2018 -0700 + + [test] disable packaging tests for suse boxes + + For #30295 + +commit c375d5ab23db63c457013c2851e915ed1043c44c +Author: Tim Brooks +Date: Thu Jul 12 11:55:38 2018 -0600 + + Add nio transport to security plugin (#31942) + + This is related to #27260. It adds the SecurityNioTransport to the + security plugin. Additionally, it adds support for ip filtering. And it + randomly uses the nio transport in security integration tests. + +commit 334c2555162910f3ffa5dcb3ed1c4c8f9940cd84 +Author: olcbean +Date: Thu Jul 12 19:10:51 2018 +0200 + + XContentTests : Insert random fields at random positions (#30867) + + Currently AbstractXContentTestCase#testFromXContent appends random fields, but in + a fixed position. This PR shuffles all fields after the random fields have been appended, + hence the random fields are actually added to random positions. + +commit 44f280fc8938039bee949bef01c16c922f1f594f +Author: Igor Motov +Date: Thu Jul 12 08:56:06 2018 -0700 + + Force execution of fetch tasks (#31974) + + Forces fetch tasks to queue even in the event that the queue is + already full. The reasoning is that fetch tasks may only be follow-up + to query tasks, so the number of additional fetch tasks that may enter + the threadpool is expected to be reasonable. + + Closes #29442 + +commit 043e40addedc8dc0b3919ee2c19a9160b3a6332e +Author: Christoph Büscher +Date: Thu Jul 12 17:38:30 2018 +0200 + + Fix unreachable error condition in AmazonS3Fixture (#32005) + + The `else` branch where currently the error response should be thrown is not + reachable because `handler` is always non-null inside the previous outer check. + Moving error creation into an else branch on the other condition check, removing + the other superflous check for non-null handler inside the first branch. + +commit ebf0dd45d2606d14d08d0ed5f6b436ed56d4e906 +Author: Colin Goodheart-Smithe +Date: Thu Jul 12 16:25:03 2018 +0100 + + Adds a check to only fail policy update if unsafe action is changed (#32002) + +commit 0b7e7befdd304a35437f9ad5f1f62a1088d85000 +Author: Alexander Reelsen +Date: Thu Jul 12 16:06:56 2018 +0200 + + Tests: Fix SearchFieldsIT.testDocValueFields (#31995) + + This test produced different implementations of joda time classes, + depending on if the data was serialized or not (DateTime vs + MutableDateTime). This now uses a common base class to extract the + milliseconds from the data. + + Closes #31992 + +commit 4ae4ac08d55d831b9e3651f2ef646a45765ceaa1 +Author: Christoph Büscher +Date: Thu Jul 12 15:50:58 2018 +0200 + + Add Expected Reciprocal Rank metric (#31891) + + This change adds Expected Reciprocal Rank (ERR) as a ranking evaluation metric + as descriped in: + + Chapelle, O., Metlzer, D., Zhang, Y., & Grinspan, P. (2009). + Expected reciprocal rank for graded relevance. + Proceeding of the 18th ACM Conference on Information and Knowledge Management. + https://doi.org/10.1145/1645953.1646033 + + ERR is an extension of the classical reciprocal rank to the graded relevance + case and assumes a cascade browsing model. It quantifies the usefulness of a + document at rank `i` conditioned on the degree of relevance of the items at ranks + less than `i`. ERR seems to be gain traction as an alternative to (n)DCG, so it + seems like a good metric to support. Also ERR seems to be the default optimization + metric used for training in RankLib, a widely used learning to rank library. + + Relates to #29653 + +commit 6fcd606536d88b1365fb3870380514f91610519b +Author: David Kyle +Date: Thu Jul 12 13:59:34 2018 +0100 + + [ML] Get ForecastRequestStats doc in RestoreModelSnapshotIT (#31973) + +commit edf83c1d87951e350be26ce86764ef783f80ab8a +Author: Andrei Stefan +Date: Thu Jul 12 15:05:42 2018 +0300 + + SQL: Add support for single parameter text manipulating functions (#31874) + + Added support for ASCII, BIT_LENGTH, CHAR, CHAR_LENGTH, LCASE, LENGTH, LTRIM, RTRIM, SPACE, UCASE functions. + Wherever Painless scripting is necessary (WHERE conditions, ORDER BY etc), those scripts are being used. + +commit 2cfe703299a9955c6eef8a1d38774c9fc63f9dc0 +Author: Dimitris Athanasiou +Date: Thu Jul 12 11:49:06 2018 +0100 + + [ML] Ensure immutability of MlMetadata (#31957) + + The test failure in #31916 revealed that updating + rules on a job was modifying the detectors list + in-place. That meant the old cluster state and the + updated cluster state had no difference and thus the + change was not propagated to non-master nodes. + + This commit fixes that and also reviews all of ML + metadata in order to ensure immutability. + + Closes #31916 + +commit 89e8f9e240fd1bbf78681e2669c8332793bb2252 +Author: Colin Goodheart-Smithe +Date: Thu Jul 12 11:40:38 2018 +0100 + + Adds concept of a safe action (#31971) + + * Adds concept of a safe action + + A safe action is one that does not have unwanted side effects if the + configuration of the action is change in the policy while and index is + executing the action. + + This commit formalises this concept with the only current unsafe action + being ShrinkAction. It also adds testing around this and add a method + to LifecyclePolicy which returns whether the action for the provided + StepKey is safe. + + * Makes IndexLifecycleRunners checks use the safe indications instead of + hardcoding shrink action + + * Fixes test failure + +commit e3707efe740356f1c5c38d71e0d98b80291f00ab +Author: Alexander Reelsen +Date: Thu Jul 12 12:17:00 2018 +0200 + + Tests: Mute SearchFieldsIT.testDocValueFields() + + Relates #31992 + +commit ae5c70e6037ed2b7f27b231177990cd590690d30 +Author: Martijn van Groningen +Date: Thu Jul 12 11:51:33 2018 +0200 + + muted tests due to #31940 + +commit eb8c82a66b0a901e5360dfd527637b3114ace376 +Author: Alpar Torok +Date: Thu Jul 12 08:55:31 2018 +0000 + + Work around reported problem in eclipse (#31960) + + The Gradle plugin + https://docs.gradle.org/current/userguide/java_gradle_plugin.html + added recently adds a folder to the CP, which is not created for the + eclipse import, ausing eclipse to complain. + +commit ba3c0f247875e8422ac023b71f4c54c4a9cec773 +Author: Alpar Torok +Date: Thu Jul 12 08:41:07 2018 +0000 + + Move build integration tests out of :buildSrc project (#31961) + + This way building buildSrc will not be delayed for all invocations, and + will be less disruptive if integration tests fail + +commit 4192f387b41fd02114bae3dba743f701e8bf61e2 +Merge: cd89ca26859 ac4e0f1b1d3 +Author: Colin Goodheart-Smithe +Date: Thu Jul 12 08:57:39 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit ac4e0f1b1d34d913b7d5394ba99794e06186d757 +Author: Alexander Reelsen +Date: Thu Jul 12 09:55:17 2018 +0200 + + Tests: Remove use of joda time in some tests (#31922) + + This also extends the dateformatters test to ensure that the printers + are acting the same in java time and joda time. + +commit 0d6b47bed9e433ccb0af52770cb662577a0e3519 +Author: Tanguy Leroux +Date: Thu Jul 12 09:21:10 2018 +0200 + + [Test] Reactive 3rd party tests on CI (#31919) + + 3rd party tests are failing because the repository-s3 is expecting some + enviromnent variables in order to test session tokens but the CI job is + not ready yet to provide those. This pull request relaxes the constraints + on the presence of env vars so that the 3rd party tests can still be + executed on CI. + + closes #31813 + +commit dc633e0000679da7ff650c9a11f53b8e17c0df96 +Author: Costin Leau +Date: Wed Jul 11 23:36:39 2018 +0300 + + SQL: Support for escape sequences (#31884) + + Enhance grammar to allow JDBC/ODBC escape sequences, namely + - date, time and timestamp {d ''}, {t ''} and {ts ''} + - guid {guid ''} + - LIKE escape {escape ''} + - scalar function {fn } + + Fix #31883 + +commit 6136e49a05f370376341d55d84ab0cd895f38a34 +Author: Costin Leau +Date: Wed Jul 11 23:31:46 2018 +0300 + + SQL: HAVING clause should accept only aggregates (#31872) + + Improve Verifier to allow HAVING clauses only on aggregates + + Close #31726 + +commit e955ffc38deea100ed535b6fefd9c8993fc6e039 +Author: Jimi Ford +Date: Wed Jul 11 15:01:49 2018 -0400 + + Docs: fix typo in datehistogram (#31972) + +commit 4d180175c67ee7c594195e7259e301b4a9a208a8 +Author: Nhat Nguyen +Date: Wed Jul 11 15:02:08 2018 -0400 + + Revert translog changes introduced for CCR (#31947) + + We introduced these changes in #26708 because for CCR. + However, CCR now uses Lucene instead of translog. + + This commit reverts these changes so that we can + minimize differences between the ccr and the master branch. + + Relates ##26708 + +commit b83e99a824cf5924eaf87beb9193c14588b9f84f +Author: Nik Everett +Date: Wed Jul 11 14:52:45 2018 -0400 + + Switch url repository rest tests to new style requests (#31944) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `module/repository-url` project to use the new + versions. + +commit 939983d783d7f17042c6cdc14bbf3e18220e3196 +Author: Nik Everett +Date: Wed Jul 11 14:42:55 2018 -0400 + + Switch reindex tests to new style requests (#31941) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `modules/reindex` project to use the new + versions. + +commit aedbfc63cdb37f8f5735b06364d3983fbcb7d92e +Author: Clinton Gormley +Date: Wed Jul 11 20:17:05 2018 +0200 + + Docs: Added note about cloud service to installation and getting started + +commit efcfd0d8275c68b9410c771aacd6781f866fb473 +Author: Lisa Cawley +Date: Wed Jul 11 09:08:32 2018 -0700 + + [DOCS] Removes alternative docker pull example (#31934) + +commit 5bcdff73d756814f270794110fb4502d0a2a513a +Author: James Baiera +Date: Wed Jul 11 12:07:31 2018 -0400 + + Add Snapshots Status API to High Level Rest Client (#31515) + + This PR adds the Snapshots Status API to the Snapshot Client, as + well as additional documentation for the status api. + +commit 51bb27a99114452c07a0b8e51d90c304f443ffb7 +Author: Jake Landis +Date: Wed Jul 11 10:13:41 2018 -0500 + + ingest: date_index_name processor template resolution (#31841) + + This change adds support for template snippet (e.g. {{foo}}) resolution + in the date_index_name processor. The following configuration options + will now resolve a templated value if so configured: + + * index_name_prefix (e.g "index_name_prefix": "myindex-{{foo}}-") + * date_rounding (e.g. "date_rounding" : "{{bar}}") + * index_name_format (e.g."index_name_format": "{{baz}}") + +commit d76293f99066db00eb121cb8b470d1f315a51263 +Author: Michael Basnight +Date: Wed Jul 11 09:37:48 2018 -0500 + + Test: fix null failure in watcher test (#31968) + + A new commit was merged that does not allow a null attachement && + text. This is valid for the slack API, as it does not allow this, but + our unit tests did. This commit fixes the broken unit test. + + Closes #31948 + +commit 38e09a1508bbef0e67ca95e46ace32f7aa1a08dc +Author: Nik Everett +Date: Wed Jul 11 10:04:17 2018 -0400 + + Switch test framework to new style requests (#31939) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `test/framework` project to use the new + versions. + +commit eda6d182b541f3b563d64956f60efa5eef9d1cd1 +Author: Nik Everett +Date: Wed Jul 11 09:48:47 2018 -0400 + + Switch low level rest tests to new style Requests (#31938) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `client/rest` project to use the new versions. + +commit aa6a1c5ca0448372953d4f117d95e9b57385182d +Author: Nik Everett +Date: Wed Jul 11 09:18:04 2018 -0400 + + Switch high level rest tests to new style requests (#31937) + + In #29623 we added `Request` object flavored requests to the low level + REST client and in #30315 we deprecated the old `performRequest`s. This + changes all calls in the `client/rest-high-level` project to use the new + versions. + +commit cd89ca268597851ceafdf26c990264b56af46c2d +Merge: 952859d87d2 b4087d69d2f +Author: Colin Goodheart-Smithe +Date: Wed Jul 11 10:26:44 2018 +0100 + + Merge branch 'master' into index-lifecycle + + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSetti + ngs.java + /Users/colings86/dev/work/git/elasticsearch/.git/worktrees/elasticsearch + -ilm/MERGE_HEAD + + buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlu + gin.groovy + client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLe + velClient.java + client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClien + t.java + client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInf + oIT.java + client/rest-high-level/src/test/java/org/elasticsearch/client/documentat + ion/MiscellaneousDocumentationIT.java + docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc + docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc + modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/Grok + ProcessorTests.java + modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/Json + ProcessorTests.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCas + e.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.j + ava + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.ja + va + qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java + qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats + server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParse + r.java + server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymPa + rser.java + server/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenF + ilterFactory.java + server/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilter + Factory.java + server/src/test/java/org/elasticsearch/index/analysis/ESSolrSynonymParse + rTests.java + server/src/test/java/org/elasticsearch/index/analysis/ESWordnetSynonymPa + rserTests.java + server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java + test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatc + her.java + test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatc + herTests.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java + x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseS + tate.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClien + tPlugin.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField + .java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSetti + ngs.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/Beat + sFeatureSetUsage.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthz/permission/Role.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/a + uthz/store/ReservedRolesStore.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/Tra + nsportXPackInfoActionTests.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/a + uthz/store/ReservedRolesStoreTests.java + x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRem + oteLicenseChecker.java + x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRem + oteLicenseCheckerTests.java + x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/au + thz/store/CompositeRolesStore.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfo + Response.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/License + Status.java -> + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/L + icenseStatus.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package + -info.java -> + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/p + ackage-info.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/packag + e-info.java -> + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/ + package-info.java + x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package + -info.java -> + x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/p + ackage-info.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfo + ResponseTests.java + x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/License + StatusTests.java -> + x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/L + icenseStatusTests.java + x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cl + uster/10_basic.yml + +commit d268b494d7e305ca65ec28b8b62380eac97dba3b +Author: David Kyle +Date: Wed Jul 11 10:17:44 2018 +0100 + + [ML] Mute test failing due to Java 11 date time format parsing bug (#31899) + +commit dadf96a840bf07b11f7dce2fcb8d6ed7dec25382 +Author: Yannick Welsch +Date: Wed Jul 11 11:12:52 2018 +0200 + + [TEST] Mute SlackMessageTests.testTemplateRender + +commit b4087d69d2f27d1640d2ada5d5f5cc0e9d05afcf +Author: Armin Braun +Date: Wed Jul 11 10:24:21 2018 +0200 + + Fix assertIngestDocument wrongfully passing (#31913) + + * Fix assertIngestDocument wrongfully passing + + * Previously docA being subset of docB passed because iteration was over docA's keys only + * Scalars in nested fields were not compared in all cases + * Assertion errors were hard to interpret (message wasn't correct since it only mentioned the class type) + * In cases where two paths contained different types a ClassCastException was thrown instead of an AssertionError + * Fixes #28492 + +commit 815faf34fc5750604341c25ddb8165ed60d496d0 +Author: Martijn van Groningen +Date: Wed Jul 11 10:16:43 2018 +0200 + + [CCR] Move api parameters from url to request body. (#31949) + + Relates to #30102 + +commit 4b8b831517f4448efa83acfe7060b0f4ee5e6beb +Author: Christoph Büscher +Date: Wed Jul 11 09:56:21 2018 +0200 + + Remove unused reference to filePermissionsCache (#31923) + + Currently Role.Builder keeps a reference to the FieldPermissionsCache that is + passed into its constructors. This seems to be unused except for passing it on + to convertFromIndicesPrivileges() in the second ctor itself, but we don't need + to keep the internal reference in that case, so it can be removed. + + Relates to #31876 + +commit c6666fc6cbb7afd3b4114453c9c9327a3361f130 +Author: Boaz Leskes +Date: Tue Jul 10 10:52:03 2018 +0200 + + rolling upgrade should use a replica to prevent relocations while running a scroll + + A scroll holds a reference to the shard store. If the cluster is moving shards + around that reference can prevent a shard from relocating back to node it used + to be on, causing test failures. + + Closes #31827 + +commit 4d83a0dd5af469cbb6ae04b1ba13deb73c617a4a +Author: Nik Everett +Date: Tue Jul 10 17:59:15 2018 -0400 + + HLREST: Bundle the x-pack protocol project (#31904) + + The `:x-pack:protocol` project is an implementation detail shared by the + xpack projects and the high level rest client and really doesn't deserve + its own maven coordinants and published javadoc. This change bundles + `:x-pack:protocol` into the high level rest client. + + Relates to #29827 + +commit 25cd8350105d62c34245e7e36636838a72b9b7ff +Author: Nhat Nguyen +Date: Tue Jul 10 17:44:26 2018 -0400 + + Increase logging level for testStressMaybeFlush + + Relates #31629 + +commit 88c270d8443c573cf593a3d7d429be5bdad14001 +Author: Sohaib Iftikhar +Date: Tue Jul 10 23:11:50 2018 +0200 + + Added lenient flag for synonym token filter (#31484) + + * Added lenient flag for synonym-tokenfilter. + + Relates to #30968 + + * added docs for synonym-graph-tokenfilter + + -- Also made lenient final + -- changed from !lenient to lenient == false + + * Changes after review (1) + + -- Renamed to ElasticsearchSynonymParser + -- Added explanation for ElasticsearchSynonymParser::add method + -- Changed ElasticsearchSynonymParser::logger instance to static + + * Added lenient option for WordnetSynonymParser + + -- also added more documentation + + * Added additional documentation + + * Improved documentation + +commit 3189ef49a5d4de87ec906764e7d703445df0c0dc +Author: Shaunak Kashyap +Date: Tue Jul 10 10:06:41 2018 -0700 + + [X-Pack] Beats centralized management: security role + licensing (#30520) + + * Adding Beats x-pack plugin + index templates + + * Adding built-in roles for Beats central management + + * Fixing typo + + * Refactoring: extract common code into method + + * More refactoring for more code reuse + + * Use a single index for Beats management + + * Rename "fragment" to "block" + + * Adding configuration block type + + * Expand kibana_system role to include Beats management index privileges + + * Fixing syntax + + * Adding test + + * Adding asserting for reserved role + + * Fixing privileges + + * Updating template + + * Removing beats plugin + + * Fixing tests + + * Fixing role variable name + + * Fixing assertions + + * Switching to preferred syntax for boolean false checks + + * Making class final + + * Making variables final + + * Updating Basic license message to be more accurate + +commit dcbb1154bf6586910721d2befb4e362e7b5f8f90 +Author: Nik Everett +Date: Tue Jul 10 13:01:28 2018 -0400 + + HLRest: Move xPackInfo() to xPack().info() (#31905) + + Originally I put the X-Pack info object into the top level rest client + object. I did that because we thought we'd like to squash `xpack` from + the name of the X-Pack APIs now that it is part of the default + distribution. We still kind of want to do that, but at least for now we + feel like it is better to keep the high level rest client aligned with + the other language clients like C# and Python. This shifts the X-Pack + info API to align with its json spec file. + + Relates to #31870 + +commit e85bb734cfc6b4b6d52f9b4e954976866d8c04cc +Author: Michael Basnight +Date: Tue Jul 10 11:17:21 2018 -0500 + + Docs: add security delete role to api call table (#31907) + +commit 006c2c9ab028c06c5199679bba7df448119e362f +Author: Andy Bristol +Date: Tue Jul 10 08:21:20 2018 -0700 + + [test] port archive distribution packaging tests (#31314) + + Recreates the rest of the bats packaging tests for the tar distribution + in the java packaging test project, with support for both tar and zip + packaging, both oss and default flavors, and on Linux and Windows. Most + tests are followed fairly closely, some have either been dropped if + unnecessary or folded into others if convenient. + +commit 952859d87d21eb9cd30f7ead393b96da206887c4 +Merge: e9cd0556c16 8ec33b742dc +Author: Tal Levy +Date: Tue Jul 10 10:55:50 2018 -0400 + + Merge branch 'master' into index-lifecycle + +commit 8ec33b742dc09eebdfba7233d28c61bcb30d5ec5 +Author: albendz <12453997+albendz@users.noreply.github.com> +Date: Tue Jul 10 07:47:00 2018 -0700 + + Watcher: Slack message empty text (#31596) + + Slack accepts an empty text or attachments, but not both. This commit + ensures that both are not empty when creating a watch. + + Closes #30071 + + Replacing old pull request: #31288 + +commit 8e1ef0cff93d30a130a106cbd4838fc68231f509 +Author: Martijn van Groningen +Date: Tue Jul 10 16:00:55 2018 +0200 + + Rewrite shard follow node task logic (#31581) + + The current shard follow mechanism is complex and does not give us easy ways the have visibility into the system (e.g. why we are falling behind). + The main reason why it is complex is because the current design is highly asynchronous. Also in the current model it is hard to apply backpressure + other than reducing the concurrent reads from the leader shard. + + This PR has the following changes: + * Rewrote the shard follow task to coordinate the shard follow mechanism between a leader and follow shard in a single threaded manner. + This allows for better unit testing and makes it easier to add stats. + * All write operations read from the shard changes api should be added to a buffer instead of directly sending it to the bulk shard operations api. + This allows to apply backpressure. In this PR there is a limit that controls how many write ops are allowed in the buffer after which no new reads + will be performed until the number of ops is below that limit. + * The shard changes api includes the current global checkpoint on the leader shard copy. This allows reading to be a more self sufficient process; + instead of relying on a background thread to fetch the leader shard's global checkpoint. + * Reading write operations from the leader shard (via shard changes api) is a separate step then writing the write operations (via bulk shards operations api). + Whereas before a read would immediately result into a write. + * The bulk shard operations api returns the local checkpoint on the follow primary shard, to keep the shard follow task up to date with what has been written. + * Moved the shard follow logic that was previously in ShardFollowTasksExecutor to ShardFollowNodeTask. + * Moved over the changes from #31242 to make shard follow mechanism resilient from node and shard failures. + + Relates to #30086 + +commit 1f0421aa69487367e7d81405acb564c27d95d46c +Author: David Kyle +Date: Tue Jul 10 11:47:47 2018 +0100 + + [ML] Mute failing DetectionRulesIT.testCondition() test + +commit 2ac7e49924df7632c96ea434563b7628b7509e50 +Author: Christoph Büscher +Date: Tue Jul 10 09:34:17 2018 +0200 + + Fix broken NaN check in MovingFunctions#stdDev() (#31888) + + The initial check will never be true, because of the special semantics of NaN, + where no value is equal to Nan, including NaN. Thus, x == Double.NaN always + evaluates to false. The method still works correct because later computations + will also return NaN if the avg argument is NaN, but the intended shortcut + doesn't work. + +commit 1c32497c44fcecaacea04170018ec62ef532b0e0 +Author: Alexander Reelsen +Date: Tue Jul 10 09:28:28 2018 +0200 + + Date: Add DateFormatters class that uses java.time (#31856) + + A newly added class called DateFormatters now contains java.time based + builders for dates, which also intends to be fully backwards compatible, + when the name based date formatters are picked. Also a new class named + CompoundDateTimeFormatter for being able to parse multiple different + formats has been added. + + A duelling test class has been added that ensures the same dates when + parsing java or joda time formatted dates for the name based dates. + + Note, that java.time and joda time are not fully backwards compatible, + which also means that old formats will currently not work with this + setup. + +commit 991d17c8c40729420fe10db11071f9a1a91c3084 +Author: David Kyle +Date: Tue Jul 10 08:26:48 2018 +0100 + + [ML] Switch native QA tests to a 3 node cluster (#31757) + +commit e60987b95fab3d2d512575a46bcead70a56b8845 +Author: Christoph Büscher +Date: Tue Jul 10 09:19:11 2018 +0200 + + Change trappy float comparison (#31889) + + Comparing primitive floats with '==' can be trappy because e.g. special handling + of NaN values. It can be avoided by using Float.compare(). + +commit a5d5234effcf711334473d1ca862c80a943bb099 +Author: Tim Vernum +Date: Tue Jul 10 11:54:24 2018 +1000 + + Fix building AD URL from domain name (#31849) + + The steps to read the settings and build URLs happen in a non-obvious + order, which meant that we would build the default URL (from the + domain name, and port) before we'd actually read the port settings. + This would cause the URL to always have a port of `0`. + + Relates: bccf988 + +commit e9cd0556c167b257bb588af5547923e5f13a99ca +Merge: 2b53510b4f2 60c1ea2c003 +Author: Tal Levy +Date: Mon Jul 9 19:44:12 2018 -0400 + + Merge branch 'master' into index-lifecycle + +commit dd21ad0d9d48642704cf64260bb5f86a75fc3803 +Author: Igor Motov +Date: Mon Jul 9 13:40:23 2018 -0700 + + Add opaque_id to audit logging (#31878) + + Logs opaque_id if it is available with all audit log messages. + + Relates #31521 + +commit 60c1ea2c00356c347aad2b15ecbdcb11704c4a95 +Author: Tal Levy +Date: Mon Jul 9 16:06:57 2018 -0400 + + re-enable backcompat tests + +commit 68a8d138286c82ce483ce96a8bc04a92452094ad +Author: Tal Levy +Date: Mon Jul 9 16:03:17 2018 -0400 + + add support for is_write_index in put-alias body parsing (#31674) + + * add support for is_write_index in put-alias body parsing + + The Rest Put-Alias Action does separate parsing of the alias body + to construct the IndicesAliasesRequest. This extra parsing + was missed in #30703. + + * test flag was not just ignored by the parser + + * disable backcompat tests + +commit 3428dc20a45695dbe52eb57f3fc8738f74a65cfc +Author: Tom Callahan +Date: Mon Jul 9 15:32:10 2018 -0400 + + Improve release notes script (#31833) + + Improve release notes script + + The current release notes script does not handle area labels that + are not two parts, such as ":ml". As these area labels are rare, + I have simply hard-coded a title for these area labels. In + addition, the script will not explicitly call out instances where + multiple area labels are present on an issue. + +commit de273651aef19be1dd8e5490fa059beca08a1759 +Author: lcawl +Date: Mon Jul 9 10:47:42 2018 -0700 + + [DOCS] Fix broken link in painless example + +commit 5481fbc249194e9e4c3f90fbdef22648cd9af816 +Author: Mayya Sharipova +Date: Mon Jul 9 11:59:49 2018 -0400 + + Handle missing values in painless (#30975) + + * Handle missing values in painless + + Throw an exception for `doc['field'].value` + if this document is missing a value for the `field`. + + For 7.0: + This is the default behaviour from 7.0 + + For 6.x: + To enable this behavior from 6.x, a user can set a jvm.option: + `-Des.script.exception_for_missing_value=true` on a node. + If a user does not enable this behavior, a deprecation warning is logged on start up. + + Closes #29286 + +commit 584fa261cc51cfca607e2068dac8f743e63496cb +Author: Jim Ferenczi +Date: Mon Jul 9 16:01:01 2018 +0200 + + Remove the ability to index or query context suggestions without context (#31007) + + This is a follow up of #30712 that removes the ability to index or query + and context enabled completion field without context. + + Relates #30712 + +commit 5f5157a2dc880cf602b29a16e3bc0ea407f36476 +Author: Armin Braun +Date: Mon Jul 9 13:50:21 2018 +0200 + + Ingest: Enable Templated Fieldnames in Rename (#31690) + + * Ingest: Enable Templated Fieldnames in Rename + +commit 5dcaac32b09d37c581f68dd8924115cf64acca11 +Author: Damien Alexandre +Date: Mon Jul 9 12:48:49 2018 +0200 + + [Docs] Fix typo in the Rollup API Quick Reference (#31855) + + The Stop endpoint was not correct in the Quick Reference. + +commit e46ed73379a70f27fadd659589a6991692289fe2 +Author: Armin Braun +Date: Mon Jul 9 10:24:34 2018 +0200 + + Ingest: Add ignore_missing option to RemoveProc (#31693) + + Added `ignore_missing` setting to the RemoveProcessor to fix #23086 + +commit c5c860dadb5c4637d4163d2a333f3e3c2eed5f83 +Author: Nicolas Ruflin +Date: Mon Jul 9 10:04:23 2018 +0200 + + Add template config for Beat state to X-Pack Monitoring (#31809) + + With https://github.com/elastic/beats/pull/7075 Beats introduces state reporting for X-Pack Monitoring. The data sent up to Elasticsearch ends up stored in the following format. + + ``` + "beats_state": { + "timestamp": "2018-07-05T07:21:03.581Z", + "state": { + "module": { + "count": 1, + "names": [ + "http" + ] + } + }, + "beat": { + "uuid": "594039b5-6353-4d78-9bad-778ecc0fe83f", + "type": "metricbeat", + "version": "7.0.0-alpha1", + "name": "ruflin", + "host": "ruflin" + } + } + ``` + + This PR adds the new fields to the template. + +commit 3aa0de9e10d33ffe0c1172b226c8eb13710228ac +Author: Alexander Reelsen +Date: Mon Jul 9 09:44:06 2018 +0200 + + Watcher: Add ssl.trust email account setting (#31684) + + In order to allow users to specify hosts, where SSL is always trusted, + this setting is exposed. Otherwise the system keystore needs to be + configured properly. + + For more info see https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html + +commit 0dac73c4fb17578aa2252035badef93d4a5bdd45 +Author: Russ Cam +Date: Fri Jul 6 15:36:31 2018 +1000 + + Remove link to oss-MSI (#31844) + + This commit removes the link to an oss-MSI; there is only one version of the MSI, which includes X-Pack. + + (cherry picked from commit d2e5db8a806ec8a25162f79db5209aceed4f30f7) + +commit 5875399738a2f23e1d7ca55ff00f6e31d5180e4b +Author: Nhat Nguyen +Date: Sun Jul 8 15:05:08 2018 -0400 + + Upgrade to Lucene-7.5.0-snapshot-b9e064b935 (#31839) + + This snapshot includes LUCENE-8384 which blocks work in Lucene rollback. + +commit d9a92011bc0ee1e43b4930462ed1ee7bd30bc370 +Author: Jack Conradson +Date: Sun Jul 8 12:00:23 2018 -0700 + + Painless: Restructure Definition/Whitelist (#31879) + + Create lookup package + rename Definition to PainlessLookup and move to lookup package + rename Definition.Method to PainlessMethod + rename Definition.MethodKey to PainlessMethod + rename Definition.Field to PainlessField + rename Definition.Struct to PainlessClass + rename Definition.Cast to PainlessCast + rename Whitelist.Struct to WhitelistClass + rename Whitelist.Constructor to WhitelistConstructor + rename Whitelist.Method to WhitelistMethod + rename Whitelist.Field to WhitelistField + +commit fb27f3e7f0388a3576738ccbc1e2995d0286fdf1 +Author: Nik Everett +Date: Sun Jul 8 11:03:56 2018 -0400 + + HLREST: Add x-pack-info API (#31870) + + This is the first x-pack API we're adding to the high level REST client + so there is a lot to talk about here! + + = Open source + + The *client* for these APIs is open source. We're taking the previously + Elastic licensed files used for the `Request` and `Response` objects and + relicensing them under the Apache 2 license. + + The implementation of these features is staying under the Elastic + license. This lines up with how the rest of the Elasticsearch language + clients work. + + = Location of the new files + + We're moving all of the `Request` and `Response` objects that we're + relicensing to the `x-pack/protocol` directory. We're adding a copy of + the Apache 2 license to the root fo the `x-pack/protocol` directory to + line up with the language in the root `LICENSE.txt` file. All files in + this directory will have the Apache 2 license header as well. We don't + want there to be any confusion. Even though the files are under the + `x-pack` directory, they are Apache 2 licensed. + + We chose this particular directory layout because it keeps the X-Pack + stuff together and easier to think about. + + = Location of the API in the REST client + + We've been following the layout of the rest-api-spec files for other + APIs and we plan to do this for the X-Pack APIs with one exception: + we're dropping the `xpack` from the name of most of the APIs. So + `xpack.graph.explore` will become `graph().explore()` and + `xpack.license.get` will become `license().get()`. + + `xpack.info` and `xpack.usage` are special here though because they + don't belong to any proper category. For now I'm just calling + `xpack.info` `xPackInfo()` and intend to call usage `xPackUsage` though + I'm not convinced that this is the final name for them. But it does get + us started. + + = Jars, jars everywhere! + + This change makes the `xpack:protocol` project a `compile` scoped + dependency of the `x-pack:plugin:core` and `client:rest-high-level` + projects. I intend to keep it a compile scoped dependency of + `x-pack:plugin:core` but I intend to bundle the contents of the protocol + jar into the `client:rest-high-level` jar in a follow up. This change + has grown large enough at this point. + + In that followup I'll address javadoc issues as well. + + = Breaking-Java + + This breaks that transport client by a few classes around. We've + traditionally been ok with doing this to the transport client. + +commit 4be6b0e2aea0a8ef1d2c69ccd18f046a33b8e06b +Merge: 6dd34345195 49ba271bd80 +Author: Nhat Nguyen +Date: Sat Jul 7 09:12:39 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [ML] Fix master node deadlock during ML daily maintenance (#31836) + Build: Switch integ-test-zip to OSS-only (#31866) + SQL: Remove restriction for single column grouping (#31818) + Build: Fix detection of Eclipse Compiler Server (#31838) + Docs: Inconsistency between description and example (#31858) + Re-enable bwc tests now that #29538 has been backported and 6.x intake build succeeded. + QA: build improvements related to SQL projects (#31862) + [Docs] Add clarification to analysis example (#31826) + Check timeZone() argument in AbstractSqlQueryRequest (#31822) + SQL: Fix incorrect HAVING equality (#31820) + Smaller aesthetic fixes to InternalTestCluster (#31831) + [Docs] Clarify accepted sort case (#31605) + Temporarily disable bwc test in order to backport #29538 + Remove obsolete parameters from analyze rest spec (#31795) + [Docs] Fix wrong link in Korean analyzer docs (#31815) + Fix profiling of ordered terms aggs (#31814) + Properly mute test involving JDK11 closes #31739 + Do not return all indices if a specific alias is requested via get aliases api. (#29538) + Get snapshot rest client cleanups (#31740) + Docs: Explain _bulk?refresh shard targeting + Fix handling of points_only with term strategy in geo_shape (#31766) + +commit 49ba271bd80b86f16eef2b59fc5561b8fbaaf77f +Author: Dimitris Athanasiou +Date: Sat Jul 7 09:43:28 2018 +0100 + + [ML] Fix master node deadlock during ML daily maintenance (#31836) + + This is the implementation for master and 6.x of #31691. + Native tests are changed to use multi-node clusters in #31757. + + Relates #31683 + +commit 20cafd31592f83092d2293e21b1a3155c9eef838 +Author: Nik Everett +Date: Fri Jul 6 14:41:19 2018 -0400 + + Build: Switch integ-test-zip to OSS-only (#31866) + + We mistakenly enabled bundling of the default distribution's bin scripts + into the `integ-test-zip` artifact used by plugin authors to test plugins. + These didn't change the version of Elasticsearch used for testing but as + a side effect changed the LICENSE.txt from the Apache 2 license to the + Elastic license. We really didn't mean for that to happen. The bin script + and the elasticsearch-sql-cli jar file bundled into the distribution are + indeed governed by the Elastic license but we didn't intend for them to be + in the testing artifact in the first place. This removes them and fixes + the license of the `integ-test-zip` artifact. + +commit 9ffb26ab02d7ea550d14f686b007be76f3149626 +Author: Costin Leau +Date: Fri Jul 6 20:55:27 2018 +0300 + + SQL: Remove restriction for single column grouping (#31818) + + For historical reasons SQL restricts GROUP BY to only one field. + This commit removes the restriction and improves the test suite with + multi group by tests. + + Close #31793 + +commit 994a251075486b7b4231634945c72c812c4dba8a +Author: Nik Everett +Date: Fri Jul 6 12:48:00 2018 -0400 + + Build: Fix detection of Eclipse Compiler Server (#31838) + + It looks like Eclipse's compiler server recently changed something so + our "eclipse detector" stopped working for it. I've updated the detector + so it ought to work now. + +commit 99030e7af5b76f425f1420897fe94cd07553bb5a +Author: Piotr Prądzyński +Date: Fri Jul 6 18:43:10 2018 +0200 + + Docs: Inconsistency between description and example (#31858) + +commit c6e7df3ac2789d54bcd54bf5ce2f93c822ccebe5 +Author: Martijn van Groningen +Date: Fri Jul 6 17:09:11 2018 +0200 + + Re-enable bwc tests now that #29538 has been backported and 6.x intake + build succeeded. + +commit 6f5d8caeb7dba372fb4a338d3c7a6d68b9781e3d +Author: Costin Leau +Date: Fri Jul 6 18:24:12 2018 +0300 + + QA: build improvements related to SQL projects (#31862) + + Register SQL as an xpackModule + Specify group for SQL QA to disambiguate projects (otherwise due to an + old Gradle bug (https://github.com/gradle/gradle/issues/847) any + subprojects under SQL QA will not be able to refer to SQL xpackModule + + Co-authored-by: Alpar Torok + +commit 3c11c7c2614ea0396f372016a5cafbc689b70ee6 +Author: Christoph Büscher +Date: Fri Jul 6 14:36:58 2018 +0200 + + [Docs] Add clarification to analysis example (#31826) + + There have been at least two PRs trying to fix the spelling of "lazi" because it + isn't very clear from the example that the english analyzer will stem each token + in the example. This adds a short description of the analysis process to make + this clearer. + + Relates to #31797 + +commit 03adbf2a39d19dde67423e8669f33bc95b59d313 +Author: hanbj +Date: Fri Jul 6 20:35:03 2018 +0800 + + Check timeZone() argument in AbstractSqlQueryRequest (#31822) + + A value of `null` will throw an IAE. + +commit 89cb0872cfcee7a715bc243b4b83ffc6f3d67239 +Author: Costin Leau +Date: Fri Jul 6 12:46:57 2018 +0300 + + SQL: Fix incorrect HAVING equality (#31820) + + Fix bug that causes `HAVING a = b` to be translated ad-litteram in + Painless which uses `==` for equality checks not `=`. + + Close #31796 + +commit cce7dc20ade5f7b6196d188915d061966969d8b5 +Author: Yannick Welsch +Date: Fri Jul 6 11:42:09 2018 +0200 + + Smaller aesthetic fixes to InternalTestCluster (#31831) + + Allows cluster to auto-reconfigure faster by starting up nodes in parallel. + +commit 450a450b2cea08c294bfb305cf72fac6a2ebc556 +Author: Christoph Büscher +Date: Fri Jul 6 10:11:36 2018 +0200 + + [Docs] Clarify accepted sort case (#31605) + + Rescore only works with an explicite "sort" element if it is on descending + "_score". Even using "order" : "asc" will throw an error. + +commit cae304406e46d348af9ded0c659dafe13dc2caa8 +Author: Martijn van Groningen +Date: Fri Jul 6 09:47:42 2018 +0200 + + Temporarily disable bwc test in order to backport #29538 + +commit d3a8094d359946ac76362b5019e4e464167392df +Author: Alan Woodward +Date: Fri Jul 6 09:05:34 2018 +0100 + + Remove obsolete parameters from analyze rest spec (#31795) + + This commit also fixes a typo in the analyze high-level client + documentation. + +commit 573613e7ca53e088e0e40f3e23e3b641f96a50d5 +Author: Jim Ferenczi +Date: Fri Jul 6 09:30:48 2018 +0200 + + [Docs] Fix wrong link in Korean analyzer docs (#31815) + +commit bf9ae5f0c8ffeb887ba251781594cd5b9627f69e +Author: Jim Ferenczi +Date: Fri Jul 6 09:00:52 2018 +0200 + + Fix profiling of ordered terms aggs (#31814) + + This change fixes the profiling of `terms` aggregation ordered + by a sub aggregation. + + Closes #22123 + +commit a81dfbcb31af5e30ff6dab1dd045ac74c6e7270b +Merge: eaa247d5437 4761a1fa29b +Author: Alpar Torok +Date: Fri Jul 6 09:59:05 2018 +0300 + + Merge branch 'master' of github.com:elastic/elasticsearch + +commit eaa247d543710c9602d0b0ff2092422d823ed656 +Author: Alpar Torok +Date: Fri Jul 6 09:58:35 2018 +0300 + + Properly mute test involving JDK11 closes #31739 + +commit 4761a1fa29b9441cc95217f912b0144c4445cedb +Author: Martijn van Groningen +Date: Fri Jul 6 08:54:32 2018 +0200 + + Do not return all indices if a specific alias is requested via get aliases api. (#29538) + + If a get alias api call requests a specific alias pattern then + indices not having any matching aliases should not be included in the response. + + This is a second attempt to fix this (first attempt was #28294). + The reason that the first attempt was reverted is because when xpack + security is enabled then index expression (like * or _all) are resolved + prior to when a request is processed in the get aliases transport action, + then `MetaData#findAliases` can't know whether requested all where + requested since it was already expanded in concrete alias names. This + change replaces aliases(...) replaceAliases(...) method on AliasesRequests + class and leave the aliases(...) method on subclasses. So there is a distinction + between when xpack security replaces aliases and a user setting aliases via + the transport or high level http client. + + Closes #27763 + +commit a5f5ea84227484e710e0764a1312ae4c80356457 +Author: Tim Brooks +Date: Thu Jul 5 17:19:46 2018 -0600 + + Get snapshot rest client cleanups (#31740) + + This is a followup to #31537. It makes a number of changes requested by + a review that came after the PR was merged. These are mostly cleanups + and doc improvements. + +commit c0b2ef55b8a5dbc38e652c8eb3a66a36e11f1acf +Author: Nik Everett +Date: Thu Jul 5 16:24:03 2018 -0400 + + Docs: Explain _bulk?refresh shard targeting + + Only the shards that receive the bulk request will be affected by + `refresh`. Imagine a `_bulk?refresh=wait_for` request with three + documents in it that happen to be routed to different shards in an index + with five shards. The request will only wait for those three shards to + refresh. The other two shards of that make up the index do not + participate in the `_bulk` request at all. + + Relates to #31819 + +commit 94d3ddd1d1ce451a76fc3edd62fc6ddf5e575b48 +Author: Igor Motov +Date: Thu Jul 5 11:40:33 2018 -0700 + + Fix handling of points_only with term strategy in geo_shape (#31766) + + Fixes 2 issues that together cause errors during index creation + with geo_shapes that use the term strategy. The term strategy changes + the default for points_only parameter, but this wasn't taken into + account during serialization. So, setting the term strategy would add + `"points_only": true` to serialization. At the same time if the term + strategy would also cause the `points_only` setting to be not marked as + a processed element during parsing, which would cause index creation to + fail with the error: `Mapping definition for [location] has unsupported` + `parameters: [points_only : true]`. + + Fixes #31707 + +commit 6dd343451958ed93362e7c49feb686dc2744c580 +Merge: f736205fd5e 09e8ac81673 +Author: Nhat Nguyen +Date: Thu Jul 5 14:07:06 2018 -0400 + + Merge branch 'master' into ccr + + * master: + REST high-level client: add get index API (#31703) + SQL: Allow long literals (#31777) + SQL: Fix incorrect message for aliases (#31792) + Test: Do not remove xpack templates when cleaning (#31642) + Reduce more raw types warnings (#31780) + Add unreleased version 6.3.2 + Scripting: Remove support for deprecated StoredScript contexts (#31394) + [ML][TEST] Use java 11 valid time format in DataDescriptionTests (#31817) + [ML] Don't treat stale FAILED jobs as OPENING in job allocation (#31800) + [ML] Fix calendar and filter updates from non-master nodes (#31804) + Fix license header generation on Windows (#31790) + mark RollupIT.testTwoJobsStartStopDeleteOne as AwaitsFix + mark SearchAsyncActionTests.testFanOutAndCollect as AwaitsFix + Correct exclusion of test on JDK 11 + Fix doclint jdk 11 + Add JDK11 support and enable in CI (#31644) + Watcher: Fix check for currently executed watches (#31137) + Watcher: Ensure correct method is used to read secure settings (#31753) + SQL: Update CLI logo + +commit 09e8ac816736f0d5eb04aa60d509426f668b11bd +Author: Sohaib Iftikhar +Date: Thu Jul 5 19:52:25 2018 +0200 + + REST high-level client: add get index API (#31703) + + Also added master_timeout parameter for the indices.get spec + + Relates to #27205 + +commit 07470c950bb67a77a05d79c607171f50aad4e3f4 +Author: Costin Leau +Date: Thu Jul 5 20:07:03 2018 +0300 + + SQL: Allow long literals (#31777) + + Fix bug that caused integral literals to be only Integer (rejecting + Long). This commit fixes that and picks either an Integer or Long based + on size. + +commit 6e9bd2652fb67fc3b76e97d273542f6213ddcab5 +Author: Costin Leau +Date: Thu Jul 5 19:49:58 2018 +0300 + + SQL: Fix incorrect message for aliases (#31792) + + * SQL: Fix incorrect message for aliases + + Fix the naming in the verification message thrown for aliases over + multiple indices with different mappings. + +commit 1099060735cb088e654586306847f5763c3c7345 +Author: Nik Everett +Date: Thu Jul 5 09:43:43 2018 -0400 + + Test: Do not remove xpack templates when cleaning (#31642) + + At the end of every `ESRestTestCase` we clean the cluster which includes + deleting all of the templates. If xpack is installed it'll automatically + recreate a few templates every time they are removed. Which is slow. + + This change stops the cleanup from removing the xpack templates. It cuts + the time to run the docs tests more than in half and it probably saves a + bit more time on other tests as well. + +commit bd1c513422d63d8f7df1fac6743ba7fd7133fa9e +Author: Christoph Büscher +Date: Thu Jul 5 15:38:06 2018 +0200 + + Reduce more raw types warnings (#31780) + + Similar to #31523. + +commit ca5822eae190c5a0cbe208f2e5d4f3d9d8a6a133 +Author: Mayya Sharipova +Date: Thu Jul 5 09:36:40 2018 -0400 + + Add unreleased version 6.3.2 + +commit 40b822c878fa058f82accbb82df9def0e9aed828 +Author: Sohaib Iftikhar +Date: Thu Jul 5 15:30:08 2018 +0200 + + Scripting: Remove support for deprecated StoredScript contexts (#31394) + + Removes support for storing scripts without the usual json around the + script. So You can no longer do: + ``` + POST _scripts/ + { + "query": { + "match": { + "title": "{{query_string}}" + } + } + } + ``` + + and must instead do: + ``` + POST _scripts/ + { + "script": { + "lang": "mustache", + "source": { + "query": { + "match": { + "title": "{{query_string}}" + } + } + } + } + } + ``` + + This improves error reporting when you attempt to store a script but don't + quite get the syntax right. Before, there was a good chance that we'd + think of it as a "raw" template and just store it. Now we won't do that. + Nice. + +commit 894fb97ad7efd6873fd6368296b3b0e7b249ecf7 +Author: Dimitris Athanasiou +Date: Thu Jul 5 13:56:54 2018 +0100 + + [ML][TEST] Use java 11 valid time format in DataDescriptionTests (#31817) + + It seems that java 11 tightened some validations with regard to + time formats. The random instance creator was setting an odd + time format to the data description which is invalid when run + with java 11. This commit changes it to a valid format. + +commit 92de94c237bf03f1a0f76560134dab90117696ca +Author: David Roberts +Date: Thu Jul 5 13:26:17 2018 +0100 + + [ML] Don't treat stale FAILED jobs as OPENING in job allocation (#31800) + + Job persistent tasks with stale allocation IDs used to always be + considered as OPENING jobs in the ML job node allocation decision. + However, FAILED jobs are not relocated to other nodes, which leads + to them blocking up the nodes they failed on after node restarts. + FAILED jobs should not restrict how many other jobs can open on a + node, regardless of whether they are stale or not. + + Closes #31794 + +commit 9c11bf1e127a488995277eaabf240c0ea3c6d195 +Author: Dimitris Athanasiou +Date: Thu Jul 5 13:14:12 2018 +0100 + + [ML] Fix calendar and filter updates from non-master nodes (#31804) + + Job updates or changes to calendars or filters may + result into updating the job process if it has been + running. To preserve the order of updates, process + updates are queued through the UpdateJobProcessNotifier + which is only running on the master node. All actions + performing such updates must run on the master node. + + However, the CRUD actions for calendars and filters + are not master node actions. They have been submitting + the updates to the UpdateJobProcessNotifier even though + it might have not been running (given the action was + run on a non-master node). When that happens, the update + never reaches the process. + + This commit fixes this problem by ensuring the notifier + runs on all nodes and by ensuring the process update action + gets the resources again before updating the process + (instead of having those resources passed in the request). + + This ensures that even if the order of the updates + gets messed up, the latest update will read the latest + state of those resource and the process will get back + in sync. + + This leaves us with 2 types of updates: + + 1. updates to the job config should happen on the master + node. This is because we cannot refetch the entire job + and update it. We need to know the parts that have been changed. + + 2. updates to resources the job uses. Those can be handled + on non-master nodes but they should be re-fetched by the + update process action. + + Closes #31803 + +commit f40581caa03191c6687b8512f5dba4d3377fed26 +Author: Costin Leau +Date: Thu Jul 5 12:28:40 2018 +0300 + + Fix license header generation on Windows (#31790) + + Updates the build.gradle to take into account the OS differences for + Windows (in particular line separator and project naming) + +commit 6acb5910128dfebe85cdced8b1eb661acedd307a +Author: Vladimir Dolzhenko +Date: Thu Jul 5 10:03:10 2018 +0200 + + mark RollupIT.testTwoJobsStartStopDeleteOne as AwaitsFix + +commit dabf2b41e08fc8443ac29b9468486170db8efe2b +Author: Vladimir Dolzhenko +Date: Thu Jul 5 09:39:54 2018 +0200 + + mark SearchAsyncActionTests.testFanOutAndCollect as AwaitsFix + +commit 1e8e3f6dae5a98b4c83a7eaa9d66ba0b47d15844 +Author: Alpar Torok +Date: Thu Jul 5 10:30:20 2018 +0300 + + Correct exclusion of test on JDK 11 + +commit d7785a7c43beb4ccc70be60ecf19b65ea5f2e9ca +Author: Alpar Torok +Date: Thu Jul 5 08:18:51 2018 +0300 + + Fix doclint jdk 11 + +commit cf2295b4084e0197df73bd0b0c4bcb012e92f31c +Author: Alpar Torok +Date: Thu Jul 5 03:24:01 2018 +0000 + + Add JDK11 support and enable in CI (#31644) + + * Upgrade bouncycastle + + Required to fix + `bcprov-jdk15on-1.55.jar; invalid manifest format ` + on jdk 11 + + * Downgrade bouncycastle to avoid invalid manifest + + * Add checksum for new jars + + * Update tika permissions for jdk 11 + + * Mute test failing on jdk 11 + + * Add JDK11 to CI + + * Thread#stop(Throwable) was removed + + http://mail.openjdk.java.net/pipermail/core-libs-dev/2018-June/053536.html + + * Disable failing tests #31456 + + * Temprorarily disable doc tests + + To see if there are other failures on JDK11 + + * Only blacklist specific doc tests + + * Disable only failing tests in ingest attachment plugin + + * Mute failing HDFS tests #31498 + + * Mute failing lang-painless tests #31500 + + * Fix backwards compatability builds + + Fix JAVA version to 10 for ES 6.3 + + * Add 6.x to bwx -> java10 + + * Prefix out and err from buildBwcVersion for readability + + ``` + > Task :distribution:bwc:next-bugfix-snapshot:buildBwcVersion + [bwc] :buildSrc:compileJava + [bwc] WARNING: An illegal reflective access operation has occurred + [bwc] WARNING: Illegal reflective access by org.codehaus.groovy.reflection.CachedClass (file:/home/alpar/.gradle/wrapper/dists/gradle-4.5-all/cg9lyzfg3iwv6fa00os9gcgj4/gradle-4.5/lib/groovy-all-2.4.12.jar) to method java.lang.Object.finalize() + [bwc] WARNING: Please consider reporting this to the maintainers of org.codehaus.groovy.reflection.CachedClass + [bwc] WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations + [bwc] WARNING: All illegal access operations will be denied in a future release + [bwc] :buildSrc:compileGroovy + [bwc] :buildSrc:writeVersionProperties + [bwc] :buildSrc:processResources + [bwc] :buildSrc:classes + [bwc] :buildSrc:jar + + ``` + + * Also set RUNTIME_JAVA_HOME for bwcBuild + + So that we can make sure it's not too new for the build to understand. + + * Align bouncycastle dependency + + * fix painles array tets + + closes #31500 + + * Update jar checksums + + * Keep 8/10 runtime/compile untill consensus builds on 11 + + * Only skip failing tests if running on Java 11 + + * Failures are dependent of compile java version not runtime + + * Condition doc test exceptions on compiler java version as well + + * Disable hdfs tests based on runtime java + + * Set runtime java to minimum supported for bwc + + * PR review + + * Add comment with ticket for forbidden apis + +commit 0a2ef59c41330bed98af1625c80f8ff1e03b3541 +Author: Alexander Reelsen +Date: Wed Jul 4 17:55:53 2018 +0200 + + Watcher: Fix check for currently executed watches (#31137) + + The ack watch action has a check for currently executed watches, to make + sure that currently running watches cannot be acknowledged. This check + only checked on the coordinating node for watches being executed, but should + have checked the whole cluster using a WatcherStatsRequest, which is + being switched to in this commit. + +commit 4328470dd80468cb411cce1bd84e8e8f27ab8295 +Author: Alexander Reelsen +Date: Wed Jul 4 15:46:59 2018 +0200 + + Watcher: Ensure correct method is used to read secure settings (#31753) + + As SecureSetting is extended from Setting, you can easily accidentally + use `SecureSetting.simpleString()` to read a secure setting instead of + `SecureSetting.secureString()`. This commit changes this behaviour in + some watcher notification services. + +commit f736205fd5ed71a26674c0932cf53901f21a8aee +Merge: ac654cbc106 308e37f80ed +Author: Nhat Nguyen +Date: Wed Jul 4 09:18:27 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [ML] Rate limit established model memory updates (#31768) + [Docs] Correct default window_size (#31582) + S3 fixture should report 404 on unknown bucket (#31782) + Detach Transport from TransportService (#31727) + [ML] Limit ML filter items to 10K (#31731) + [ML] Return statistics about forecasts as part of the jobsstats and usage API (#31647) + Fixture for Minio testing (#31688) + [DOCS] Add missing get mappings docs to HLRC (#31765) + [DOCS] Starting Elasticsearch (#31701) + Painless: Complete Removal of Painless Type (#31699) + Fix not waiting for Netty ThreadDeathWatcher in IT (#31758) + Consolidate watcher setting update registration (#31762) + Build: re-enabled bwc (#31769) + ingest: Introduction of a bytes processor (#31733) + Fix coerce validation_method in GeoBoundingBoxQueryBuilder (#31747) + Add analyze API to high-level rest client (#31577) + [DOCS] Typos + DOC: Add examples to the SQL docs (#31633) + Add support for AWS session tokens (#30414) + Watcher: Reenable start/stop yaml tests (#31754) + Implemented XContent serialisation for GetIndexResponse (#31675) + JDBC: Fix stackoverflow on getObject and timestamp conversion (#31735) + resolveHasher defaults to NOOP (#31723) + Account for XContent overhead in in-flight breaker + Split CircuitBreaker-related tests (#31659) + Add write*Blob option to replace existing blob (#31729) + Painless: Add Context Docs (#31190) + Watcher: Fix chain input toXcontent serialization (#31721) + Docs: Match the examples in the description (#31710) + rest-high-level: added get cluster settings (#31706) + [Docs] Correct typos (#31720) + Clean up double semicolon code typos (#31687) + [DOCS] Check for Windows and *nix file paths (#31648) + [ML] Validate ML filter_id (#31535) + Revert long lines + Fix TransportChangePasswordActionTests + +commit 79193727bd2171c30165ce9648ce33085b5ac0f8 +Author: Costin Leau +Date: Wed Jul 4 15:54:22 2018 +0300 + + SQL: Update CLI logo + +commit 308e37f80ed474c86546d3b319e439b7da3c810a +Author: David Roberts +Date: Wed Jul 4 13:56:32 2018 +0100 + + [ML] Rate limit established model memory updates (#31768) + + There is at most one model size stats document per bucket, but + during lookback a job can churn through many buckets very quickly. + This can lead to many cluster state updates if established model + memory needs to be updated for a given model size stats document. + + This change rate limits established model memory updates to one + per job per 5 seconds. This is done by scheduling the updates 5 + seconds in the future, but replacing the value to be written if + another model size stats document is received during the waiting + period. Updating the values in arrears like this means that the + last value received will be the one associated with the job in the + long term, whereas alternative approaches such as not updating the + value if a new value was close to the old value would not. + +commit 5f87a84bef93dfd53b696cc6deac8edef22f5258 +Author: Christoph Büscher +Date: Wed Jul 4 14:07:20 2018 +0200 + + [Docs] Correct default window_size (#31582) + +commit 18c17dfb35ba84ceb3fb78e166d73d9462ef8290 +Author: David Turner +Date: Wed Jul 4 10:48:00 2018 +0100 + + S3 fixture should report 404 on unknown bucket (#31782) + + Today, `AmazonS3Fixture` returns 403 on attempts to access any inappropriate + bucket, whether known or otherwise. In fact, S3 reports 404 on nonexistent + buckets and 403 on inaccessible ones. This change enhances `AmazonS3Fixture` to + distinguish these cases. + +commit 3f2a241b7f069b2a4c274dcc0b9b7903a5634933 +Author: Simon Willnauer +Date: Wed Jul 4 11:32:35 2018 +0200 + + Detach Transport from TransportService (#31727) + + Today TransportService is tightly coupled with Transport since it + requires an instance of TransportService in order to receive responses + and send requests. This is mainly due to the Request and Response handlers + being maintained in TransportService but also because of the lack of a proper + callback interface. + + This change moves request handler registry and response handler registration into + Transport and adds all necessary methods to `TransportConnectionListener` in order + to remove the `TransportService` dependency from `Transport` + Transport now accepts one or more `TransportConnectionListener` instances that are + executed sequentially in a blocking fashion. + +commit 896317fe36d8dbc236a837fb07704fb8e26dcd4d +Author: Dimitris Athanasiou +Date: Wed Jul 4 10:25:00 2018 +0100 + + [ML] Limit ML filter items to 10K (#31731) + + Add hard limit to the number of items + a filter may have. This serves to protect + from excessive overhead due to the filters + taking too much memory or lookups becoming + too expensive. + +commit e9f8442beeea4bcd34d442a5ee7e523779dccaf2 +Author: Hendrik Muhs +Date: Wed Jul 4 08:15:45 2018 +0200 + + [ML] Return statistics about forecasts as part of the jobsstats and usage API (#31647) + + This change adds stats about forecasts, to the jobstats api as well as xpack/_usage. The following + information is collected: + + _xpack/ml/anomaly_detectors/{jobid|_all}/_stats: + + - total number of forecasts + - memory statistics (mean/min/max) + - runtime statistics + - record statistics + - counts by status + + _xpack/usage + + - collected by job status as well as overall (_all): + - total number of forecasts + - number of jobs that have at least 1 forecast + - memory, runtime, record statistics + - counts by status + + Fixes #31395 + +commit 32d67ef5048a6c72fa378bb6c8c5fdfb2c0a13e4 +Author: Yannick Welsch +Date: Wed Jul 4 08:04:07 2018 +0200 + + Fixture for Minio testing (#31688) + + Adds a Minio fixture to run the S3 repository tests against Minio. Also collapses the single qa + subproject into the s3-repository project, which simplifies the code structure (having it all in one + place) and helps to avoid having too many Gradle subprojects. + +commit 42ff8aa2ef21d28d54d87b76f7e746a9f15334e8 +Author: Lee Hinman +Date: Tue Jul 3 17:10:52 2018 -0600 + + [DOCS] Add missing get mappings docs to HLRC (#31765) + + This commit adds the high-level rest client docs for the get mappings API that + was added in #30889 + +commit ac7fadd3367aa70ba256a1133942a76c9f6ab333 +Author: Lisa Cawley +Date: Tue Jul 3 13:40:37 2018 -0700 + + [DOCS] Starting Elasticsearch (#31701) + +commit a02e5ee740f663601e9cea32a785ef252d28a6b8 +Author: Jack Conradson +Date: Tue Jul 3 13:31:56 2018 -0700 + + Painless: Complete Removal of Painless Type (#31699) + + This completes the removal of Painless Type. The new data structures in the definition are a map of names (String) to Java Classes and a map of Java Classes to Painless Structs. The names to Java Classes map can contain a 2 to 1 ratio of names to classes depending on whether or not a short (imported) name is used. The Java Classes to Painless Structs is 1 to 1 always where the Java Class name must match the Painless Struct name. This should lead a significantly simpler type system in Painless moving forward since the Painless Type only held redundant information since Painless does not support generics. + +commit ed41d4f5661264ce7d9fb8fcdd21bc9a5a722fd2 +Author: Armin Braun +Date: Tue Jul 3 18:55:23 2018 +0200 + + Fix not waiting for Netty ThreadDeathWatcher in IT (#31758) + + Same problem and solution as in #30763 + Fixes #30547 + +commit e65115ae5ae0e07af986ef109bb3fde5af644b47 +Author: Michael Basnight +Date: Tue Jul 3 11:29:39 2018 -0500 + + Consolidate watcher setting update registration (#31762) + + Previously the call to register a listener for settings updates was in + each individual service, rather than in the notification service + itself. This change ensures that each child of the notification service + gets registered with the settings update consumer. + +commit dc869aa14993e29e080bfa3bc0ff75a057c3af92 +Author: Sohaib Iftikhar +Date: Tue Jul 3 18:13:28 2018 +0200 + + Build: re-enabled bwc (#31769) + + -- It was disabled by #31675 + +commit c0056cddd8ab0317757d41ee1fa651b0c95617c5 +Author: Jake Landis +Date: Tue Jul 3 10:40:56 2018 -0500 + + ingest: Introduction of a bytes processor (#31733) + + ingest: Introduction of a bytes processor + + This processor allows for human readable byte values (e.g. 1kb) to be converted to value in bytes (e.g. 1024). Internally this processor re-uses "ByteSizeValue.parseBytesSizeValue" which supports conversions up to Long.MAX_VALUE and the following units: "b", "kb", "mb", "gb", "tb", pb". + + This change also introduces a generic return type for the AbstractStringProcessor to allow for code reuse while supporting a String -> T conversion. (String -> Long in this case). + +commit 396c5780667121266a2e609383ff2d2d8111d469 +Author: Igor Motov +Date: Tue Jul 3 08:08:40 2018 -0700 + + Fix coerce validation_method in GeoBoundingBoxQueryBuilder (#31747) + + The Rectangle constructor validates bounds before coerce has a chance + to normalize coordinates so it cannot be used as intermittent storage. + This commit removes the Rectangle as an intermittent storage for the + bounding box coordinates. + + Fixes #31718 + +commit 1d114071dabc8b5208c797a9bbc3b990680577b7 +Author: Alan Woodward +Date: Tue Jul 3 15:57:02 2018 +0100 + + Add analyze API to high-level rest client (#31577) + +commit 093ea037b4cb6a747583a35e53b4df740fd4a36e +Author: Costin Leau +Date: Tue Jul 3 17:19:48 2018 +0300 + + [DOCS] Typos + +commit de9e56aa0161abb30047e7d96b814ad250b69165 +Author: Costin Leau +Date: Tue Jul 3 16:56:31 2018 +0300 + + DOC: Add examples to the SQL docs (#31633) + + Significantly improve the example snippets in the documentation. + The examples are part of the test suite and checked nightly. + To help readability, the existing dataset was extended (test_emp renamed + to emp plus library). + Improve output of JDBC tests to be consistent with the CLI + Add lenient flag to JDBC asserts to allow type widening (a long is + equivalent to a integer as long as the value is the same). + +commit 4108722052707c4d9acc1b211e6fd2f77885dcc2 +Author: David Turner +Date: Tue Jul 3 14:12:07 2018 +0100 + + Add support for AWS session tokens (#30414) + + AWS supports the creation and use of credentials that are only valid for a + fixed period of time. These credentials comprise three parts: the usual access + key and secret key, together with a session token. This commit adds support for + these three-part credentials to the EC2 discovery plugin and the S3 repository + plugin. + + Note that session tokens are only valid for a limited period of time and yet + there is no mechanism for refreshing or rotating them when they expire without + restarting Elasticsearch. Nonetheless, this feature is already useful for + nodes that need only run for a few days, such as for training, testing or + evaluation. #29135 tracks the work towards allowing these credentials to be + refreshed at runtime. + + Resolves #16428 + +commit 69f8934101f495143f09da0e11a6810561f18d58 +Author: Alexander Reelsen +Date: Tue Jul 3 14:52:52 2018 +0200 + + Watcher: Reenable start/stop yaml tests (#31754) + + The underlying cause for this has been fixed, thus the tests can be + reenabled. + + Closes #30298 + +commit a5fd4a77098ff533c636f6be779ccd90965967f5 +Author: Sohaib Iftikhar +Date: Tue Jul 3 14:08:50 2018 +0200 + + Implemented XContent serialisation for GetIndexResponse (#31675) + + This PR does the server side work for adding the Get Index API to the REST + high-level-client, namely moving resolving default settings to the + transport action. A follow up would be the client side changes. + +commit ac654cbc10684c39145e5a9dbaae7b112fa1dd81 +Author: Martijn van Groningen +Date: Tue Jul 3 13:15:06 2018 +0200 + + Follow engine should not fill gaps upon promotion and recovery (#31751) + + Closes #31318 + +commit ce78925732f17a3628a356133cbd75b71fc79c57 +Author: Andrei Stefan +Date: Tue Jul 3 13:55:33 2018 +0300 + + JDBC: Fix stackoverflow on getObject and timestamp conversion (#31735) + + StackOverflowError fix in JdbcResultSet getObject method. + Fix Timestamp conversion bug when getting the value of a time column. + +commit 49b977ba7ca482d518f3001221a117baa3ffe2ae +Author: Ioannis Kakavas +Date: Tue Jul 3 11:31:48 2018 +0300 + + resolveHasher defaults to NOOP (#31723) + + * Default resolveFromHash to Hasher.NOOP + + This changes the default behavior when resolving the hashing + algorithm from unrecognised hash strings, which was introduced in + #31234 + + A hash string that doesn't start with an algorithm identifier can + either be a malformed/corrupted hash or a plaintext password when + Hasher.NOOP is used(against warnings). + Do not make assumptions about which of the two is true for such + strings and default to Hasher.NOOP. Hash verification will subsequently + fail for malformed hashes. + Finally, do not log the potentially malformed hash as this can very + well be a plaintext password. + + Resolves #31697 + Reverts 58cf95a06f1defd31b16c831708ca32a5b445f98 + +commit 3d53daeb2f58c5b6986f80a238f8b28d90447f88 +Author: Daniel Mitterdorfer +Date: Tue Jul 3 09:17:16 2018 +0200 + + Account for XContent overhead in in-flight breaker + + So far the in-flight request circuit breaker has only accounted for the + on-the-wire representation of a request. However, we convert the raw + request into XContent internally which increases the overhead. + Therefore, we increase the value of the corresponding setting + `network.breaker.inflight_requests.overhead` from one to two. While this + value is still rather conservative (we assume that the representation as + structured objects has no overhead compared to the byte[]), it is closer + to reality than the current value. + + Relates #31613 + +commit ee4dbc8dedaea2323696057ed44c7dc2cb9fc0d9 +Author: Daniel Mitterdorfer +Date: Tue Jul 3 09:16:26 2018 +0200 + + Split CircuitBreaker-related tests (#31659) + + `MemoryCircuitBreakerTests` conflates two test aspects: It tests + individual circuit breakers as well as the circuit breaker hierarchy. + With this commit we split those two aspects into two test classes: + + * Tests for individual circuit breakers stay in the current class + * Other tests are moved to `HierarchyCircuitBreakerServiceTests` + +commit 2bb4f3837113952488a9c71773107b024b756d14 +Author: Yannick Welsch +Date: Tue Jul 3 09:13:50 2018 +0200 + + Add write*Blob option to replace existing blob (#31729) + + Adds a new parameter to the BlobContainer#write*Blob methods to specify whether the existing file + should be overridden or not. For some metadata files in the repository, we actually want to replace + the current file. This is currently implemented through an explicit blob delete and then a fresh write. + In case of using a cloud provider (S3, GCS, Azure), this results in 2 API requests instead of just 1. + This change will therefore allow us to achieve the same functionality using less API requests. + +commit 631a53a0e1d6be928f6c7bb55bc4c64de5bba695 +Author: Jack Conradson +Date: Mon Jul 2 14:44:36 2018 -0700 + + Painless: Add Context Docs (#31190) + + Adds documentation for each the variables and API available with each script context + usable with a Painless script. + +commit 8f2feb84143aff46225f0bca84d6f7d0d40f99fc +Author: Alexander Reelsen +Date: Mon Jul 2 20:35:26 2018 +0200 + + Watcher: Fix chain input toXcontent serialization (#31721) + + The xcontent parameters were not passed to the xcontent serialization + of the chain input for each chain. This could lead to wrongly stored + watches, which did not contain passwords but only their redacted counterparts, when an input inside of a chain input contained a password. + +commit ea15284230fbbb574b57ee5da7d49ed44c3cb152 +Author: Peter Evers +Date: Mon Jul 2 20:12:49 2018 +0200 + + Docs: Match the examples in the description (#31710) + + Prose drifted from snippet. + +commit c55d11f8b547cf959c56805f2ecc15e6772d9e5d +Author: Sohaib Iftikhar +Date: Mon Jul 2 19:25:17 2018 +0200 + + rest-high-level: added get cluster settings (#31706) + + Relates to #27205 + +commit 1dd10fe69f75c979292f3f0db27ef6671d8cce54 +Author: Stéphane Campinas +Date: Mon Jul 2 15:17:31 2018 +0200 + + [Docs] Correct typos (#31720) + +commit 31aabe4bf9ea73cbb1c21322d9e9aa2a578b41a0 +Author: Christoph Büscher +Date: Mon Jul 2 15:14:44 2018 +0200 + + Clean up double semicolon code typos (#31687) + +commit 5d94003dc028280cf80c9b38e373dc73a5c511ba +Author: David Kyle +Date: Mon Jul 2 13:10:52 2018 +0100 + + [DOCS] Check for Windows and *nix file paths (#31648) + + Proper cleanup of the docs snippet tests depends on detecting what is being tested (ML, Watcher, etc) this is deduced from the file path and so we must account for Windows and Unix path separators + +commit 8e838ea12e37ae10db6f844a8bcf4a1158c89976 +Author: Dimitris Athanasiou +Date: Mon Jul 2 12:48:04 2018 +0100 + + [ML] Validate ML filter_id (#31535) + + Like job and datafeed ids, the filter id + should be validated with the same rules + to avoid document ids that can be problematic. + +commit 3baaa8012e4dc74a12229e828a0e189fff9618ce +Author: Ioannis Kakavas +Date: Mon Jul 2 12:50:18 2018 +0300 + + Revert long lines + + Introduced in a7eaa409e804f218aa06fd02d9166b9a5998b48a + +commit a7eaa409e804f218aa06fd02d9166b9a5998b48a +Author: Ioannis Kakavas +Date: Mon Jul 2 10:38:40 2018 +0300 + + Fix TransportChangePasswordActionTests + + testIncorrectPasswordHashingAlgorithm is based on the assumption + that the algorithm selected for the change password request is + different than the one selected for the NativeUsersStore. + pbkdf2_10000 is the same as pbkdf2 since 10000 is the default cost + factor for pbkdf2 and thus should not be used as an option for the + passwordHashingSettings. + + Also make sure that the same algorithm is used for settings and + change password requests in other tests for consistency, even if + we expect to not reach the code where the algorithm is checked for + now. + + Resolves #31696 + Reverts 1c4f480794f2465c78e8e29645956f16971eeead + +commit 05b4517f2f96ccb8d69d2a3a775d893b8d64c6aa +Merge: e46d23dfb2b 2971dd56ca5 +Author: Jason Tedor +Date: Sun Jul 1 14:42:04 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: + Enable setting client path prefix to / (#30119) + [DOCS] Secure settings specified per node (#31621) + has_parent builder: exception message/param fix (#31182) + +commit 2971dd56ca58ee82a43c15c03e0e977718bf71df +Author: Konrad Beiske +Date: Sun Jul 1 19:42:03 2018 +0200 + + Enable setting client path prefix to / (#30119) + + Some proxies require all requests to have paths starting with / since + there are no relative paths at the HTTP connection level. Elasticsearch + assumes paths are absolute. In order to run rest tests against a cluster + behind such a proxy, set the system property + tests.rest.client_path_prefix to /. + +commit 85ec4970561c1a755507801a1eb8244a8c67a729 +Author: Albert Zaharovits +Date: Sun Jul 1 11:11:47 2018 +0300 + + [DOCS] Secure settings specified per node (#31621) + + Make it clear that secure settings have to be set + on each cluster node. + +commit c827a4e8e1d65e9158b5e4889a3add9f34002ee6 +Author: Nirmal Chidambaram +Date: Sat Jun 30 18:17:37 2018 +0000 + + has_parent builder: exception message/param fix (#31182) + + has_parent builder throws exception message that it expects a `type` + while parser excepts `parent_type` + +commit e46d23dfb2bd4c0677933917e77f7a63cfb5db01 +Merge: 540da7399c5 1a54bca7126 +Author: Nhat Nguyen +Date: Fri Jun 29 19:20:15 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Mute 'Test typed keys parameter for suggesters' as we await a fix. + Build test: Thread linger + Fix gradle4.8 deprecation warnings (#31654) + Mute FileRealmTests#testAuthenticateCaching with an @AwaitsFix. + Mute TransportChangePasswordActionTests#testIncorrectPasswordHashingAlgorithm with an @AwaitsFix. + Build: Fix naming conventions task (#31681) + Introduce a Hashing Processor (#31087) + +commit 540da7399c59ed6bec350add622317bc2effc831 +Author: Nhat Nguyen +Date: Fri Jun 29 17:35:21 2018 -0400 + + TEST: Randomize soft-deletes settings (#31585) + + This change allows us to test our system with/without soft-deletes enabled. + +commit 1a54bca71264cb050a354b8129100546c9caa631 +Author: Julie Tibshirani +Date: Fri Jun 29 12:57:41 2018 -0700 + + Mute 'Test typed keys parameter for suggesters' as we await a fix. + +commit 144735e62a3505477ad9d743ef0f9960e89ec08e +Author: Nik Everett +Date: Fri Jun 29 15:17:54 2018 -0400 + + Build test: Thread linger + + Add a thread linger filter to the build test so we don't spuriously fail + waiting on the "connection worker" thread. + +commit 200e1f45f28f3cedb5050bdb661dae8c412f6811 +Author: Alpar Torok +Date: Fri Jun 29 19:17:19 2018 +0000 + + Fix gradle4.8 deprecation warnings (#31654) + + * remove explicit wrapper task + + It's created by Gradle and triggers a deprecation warning + Simplify configuration + + * Upgrade shadow plugin to get rid of Gradle deprecation + + * Move compile configuration to base plugin + + Solves Gradle deprecation warning from earlier Gradle versions + + * Enable stable publishing in the Gradle build + + * Replace usage of deprecated property + + * bump Gradle version in build compare + +commit 58cf95a06f1defd31b16c831708ca32a5b445f98 +Author: Julie Tibshirani +Date: Fri Jun 29 12:02:29 2018 -0700 + + Mute FileRealmTests#testAuthenticateCaching with an @AwaitsFix. + +commit 1c4f480794f2465c78e8e29645956f16971eeead +Author: Julie Tibshirani +Date: Fri Jun 29 11:45:50 2018 -0700 + + Mute TransportChangePasswordActionTests#testIncorrectPasswordHashingAlgorithm with an @AwaitsFix. + +commit f702086df16ed42d565438c54bd9dcb151ba20d5 +Author: Alpar Torok +Date: Fri Jun 29 18:41:02 2018 +0000 + + Build: Fix naming conventions task (#31681) + + Move min compiler and runtime version to files so we can fix the source + and target comparability of the build script to those + + Closes #31665 + +commit 8c78fe711486a3f722879d8f7cf89632daf793e0 +Author: Tal Levy +Date: Fri Jun 29 09:30:54 2018 -0700 + + Introduce a Hashing Processor (#31087) + + It is useful to have a processor similar to + logstash-filter-fingerprint + in Elasticsearch. A processor that leverages a variety of hashing algorithms + to create cryptographically-secure one-way hashes of values in documents. + + This processor introduces a pbkdf2hmac hashing scheme to fields in documents + for indexing + +commit 52d9012d31339fbb6d6e949f41c2114dacfddec0 +Merge: 8ecfcc3b807 8fa06294abe +Author: Nhat Nguyen +Date: Fri Jun 29 10:12:03 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Do not check for object existence when deleting repository index files (#31680) + Remove extra check for object existence in repository-gcs read object (#31661) + Support multiple system store types (#31650) + [Test] Clean up some repository-s3 tests (#31601) + [Docs] Use capital letters in section headings (#31678) + [DOCS] Add PQL language Plugin (#31237) + Merge AzureStorageService and AzureStorageServiceImpl and clean up tests (#31607) + TEST: Fix test task invocation (#31657) + Revert "[TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg" + Fix RealmInteg test failures + Extend allowed characters for grok field names (#21745) (#31653) + [DOCS] Fix licensing API details (#31667) + [TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg + Fix CreateSnapshotRequestTests Failure (#31630) + Configurable password hashing algorithm/cost (#31234) + [TEST] Mute failing NamingConventionsTaskIT tests + [DOCS] Replace CONFIG_DIR with ES_PATH_CONF (#31635) + Core: Require all actions have a Task (#31627) + +commit 8fa06294abe5f861db55c2cee04053c2fe285655 +Author: Tanguy Leroux +Date: Fri Jun 29 14:15:34 2018 +0200 + + Do not check for object existence when deleting repository index files (#31680) + + Before deleting a repository index generation file, BlobStoreRepository + checks for the existence of the file and then deletes it. We can save + a request here by using BlobContainer.deleteBlobIgnoringIfNotExists() + which ignores error when deleting a file that does not exist. + + Since there is no way with S3 to know if a non versioned file existed + before being deleted, this pull request also changes S3BlobContainer so + that it now implements deleteBlobIgnoringIfNotExists(). It will now save + one more request (blobExist?) when appropriate. The tests and fixture + have been modified to conform the S3 API that always returns a 204/NO + CONTENT HTTP response on deletions. + +commit d8b3f332ef2adcfc7bc80b87189f08f1e1f6d655 +Author: Tanguy Leroux +Date: Fri Jun 29 13:52:31 2018 +0200 + + Remove extra check for object existence in repository-gcs read object (#31661) + +commit 117e9066db785182335ac74c8bacad30858c222f +Author: Ioannis Kakavas +Date: Fri Jun 29 14:41:13 2018 +0300 + + Support multiple system store types (#31650) + + Support multiple system store types + + When falling back to using the system keystore and - most usually - + truststore, do not assume that it will be a JKS store, but deduct + its type from {@code KeyStore#getDefaultKeyStoreType}. This allows + the use of any store type the Security Provider supports by setting + the keystore.type java security property. + +commit 0ef22db844467d84ce3d273a67276cfda7f2b852 +Author: Tanguy Leroux +Date: Fri Jun 29 13:21:29 2018 +0200 + + [Test] Clean up some repository-s3 tests (#31601) + + This commit removes some tests in the repository-s3 plugin that + have not been executed for 2+ years but have been maintained + for nothing. Most of the tests in AbstractAwsTestCase were + obsolete or superseded by fixture based integration tests. + +commit ffc8b82ea34697ee14173268740431f585adf5a6 +Author: Fredrik Meyer +Date: Fri Jun 29 11:56:33 2018 +0200 + + [Docs] Use capital letters in section headings (#31678) + + Section headings should start with capital letters. + +commit 8ecfcc3b80701668449959bd9e33de6d34a175e5 +Author: Martijn van Groningen +Date: Fri Jun 29 11:47:46 2018 +0200 + + muted tests that will be replaced by the shard follow task refactoring: + https://github.com/elastic/elasticsearch/pull/31581 + +commit 2a3a86bb5ec3857576fa8eec301d83e0a749be9f +Author: ritesh-kapoor +Date: Fri Jun 29 15:05:18 2018 +0530 + + [DOCS] Add PQL language Plugin (#31237) + + Add PQL language Plugin to community plugin page + +commit 7a0a0f2a9ec593632757eebf49d4d73bf7fb7573 +Author: Tanguy Leroux +Date: Fri Jun 29 11:04:35 2018 +0200 + + Merge AzureStorageService and AzureStorageServiceImpl and clean up tests (#31607) + + This pull request merges the AzureStorageService interface and + the AzureStorageServiceImpl classes into one single + AzureStorageService class. It also removes some tests in the + repository-azure plugin that have not been executed for 2+ years. + +commit 7a76e3a4fa516baba969c5a4d79e14af4c7ef454 +Author: Armin Braun +Date: Fri Jun 29 10:06:36 2018 +0200 + + TEST: Fix test task invocation (#31657) + +commit 915dcec154c9ae6467a4770aa76fdcad1b4858fc +Author: Ioannis Kakavas +Date: Fri Jun 29 10:34:58 2018 +0300 + + Revert "[TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg" + + This reverts commit eef8e803e54b8adffaf446e2712c6215499a6e7e. + +commit 40bf58e2a035dd497a19ed4f87b3741e18806301 +Author: Ioannis Kakavas +Date: Fri Jun 29 10:25:45 2018 +0300 + + Fix RealmInteg test failures + + As part of the changes in #31234,the password verification logic + determines the algorithm used for hashing the password from the + format of the stored password hash itself. Thus, it is generally + possible to validate a password even if it's associated stored hash + was not created with the same algorithm than the one currently set + in the settings. + At the same time, we introduced a check for incoming client change + password requests to make sure that the request's password is hashed + with the same algorithm that is configured to be used in the node + settings. + In the spirit of randomizing the algorithms used, the + {@code SecurityClient} used in the {@code NativeRealmIntegTests} and + {@code ReservedRealmIntegTests} would send all requests dealing with + user passwords by randomly selecting a hashing algorithm each time. + This meant that some change password requests were using a different + password hashing algorithm than the one used for the node and the + request would fail. + This commit changes this behavior in the two aforementioned Integ + tests to use the same password hashing algorithm for the node and the + clients, no matter what the request is. + + Resolves #31670 + +commit b7b413e55e21bd799852318cf52c96d802f74702 +Author: Armin Braun +Date: Fri Jun 29 09:12:47 2018 +0200 + + Extend allowed characters for grok field names (#21745) (#31653) + +commit 5925611e9ed8f5513fc3b0d0904b750e945cc6c9 +Author: Lisa Cawley +Date: Thu Jun 28 15:38:41 2018 -0700 + + [DOCS] Fix licensing API details (#31667) + +commit 573df2dd2e52442d7d25d73566f5807aaf86828e +Author: Nhat Nguyen +Date: Thu Jun 28 17:42:14 2018 -0400 + + Relax TranslogWriter#assertNoSeqNumberConflict (#31569) + + If the recovery and indexing are concurrently happening, it's possible + for a replica to receive the same operation twice: one from the + replication, and the other from the recovery. However, these operations + are not identical because we don't store the versionType of operations + in the Lucene index. + + The TranslogWriter#assertNoSeqNumberConflict assertion has been tripped + several times in the CCR branch since we use Lucene in peer-recovery. + This commit relaxes that assertion by excluding the versionType from the + check. + +commit eef8e803e54b8adffaf446e2712c6215499a6e7e +Author: Zachary Tong +Date: Thu Jun 28 21:32:27 2018 +0000 + + [TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg + + Tracking issue: #31670 + +commit 02c01cb4cf1f9fb4d5d6067ce0a0654df91d2af7 +Author: Jack Conradson +Date: Thu Jun 28 12:12:55 2018 -0700 + + Fix CreateSnapshotRequestTests Failure (#31630) + + Original test failure found here in issue (#31625). Had to rework the tests to only include options available externally for create snapshot requests. + +commit db6b33978e613f27f1d71178649a3e528972bff7 +Author: Ioannis Kakavas +Date: Thu Jun 28 19:39:39 2018 +0300 + + Configurable password hashing algorithm/cost (#31234) + + Make password hashing algorithm/cost configurable for the + stored passwords of users for the realms that this applies + (native, reserved). Replaces predefined choice of bcrypt with + cost factor 10. + This also introduces PBKDF2 with configurable cost + (number of iterations) as an algorithm option for password hashing + both for storing passwords and for the user cache. + Password hash validation algorithm selection takes into + consideration the stored hash prefix and only a specific number + of algorithnm and cost factor options for brypt and pbkdf2 are + whitelisted and can be selected in the relevant setting. + +commit 1af31f441ab05cc3b422cd22a9633ef070f4cdd8 +Author: Zachary Tong +Date: Thu Jun 28 16:22:35 2018 +0000 + + [TEST] Mute failing NamingConventionsTaskIT tests + + Note: no awaitsFix available, so I fell back to JUnit @Ignore + + Tracking issue: #31665 + +commit 101d675f904a13caa84bae891aebdfc6e45a8445 +Author: Lisa Cawley +Date: Thu Jun 28 08:27:04 2018 -0700 + + [DOCS] Replace CONFIG_DIR with ES_PATH_CONF (#31635) + +commit 1185ddbcc65f10a6bb876f035ab7dbbe10f11169 +Author: Nhat Nguyen +Date: Thu Jun 28 11:24:41 2018 -0400 + + Replaces testClassesDir with testClassesDirs in ccr build + + Relates #30389 + +commit f9248352656f80a355f3da374ea87d705647bd41 +Author: Ryan Ernst +Date: Thu Jun 28 08:24:03 2018 -0700 + + Core: Require all actions have a Task (#31627) + + The TaskManager and TaskAwareRequest could return null when registering + a task according to their javadocs, but no implementations ever actually + did that. This commit removes that wording from the javadocs and ensures + null is no longer allowed. + +commit 2c10cd3e1dfaef5054d44a373c63ec22c7da962a +Merge: e4315cc7106 0522c6644d3 +Author: Nhat Nguyen +Date: Thu Jun 28 11:13:01 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Docs: Remove duplicate test setup + Print output when the name checker IT fails (#31660) + Fix syntax errors in get-snapshots docs (#31656) + Docs: Fix description of percentile ranks example example (#31652) + Add MultiSearchTemplate support to High Level Rest client (#30836) + Add test for low-level client round-robin behaviour (#31616) + SQL: Refactor package names of sql-proto and sql-shared-proto projects (#31622) + Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389) + Correct integTest enable logic (#31646) + Fix missing get-snapshots docs reference #31645 + Do not check for Azure container existence (#31617) + Merge AwsS3Service and InternalAwsS3Service in a S3Service class (#31580) + Upgrade gradle wrapper to 4.8 (#31525) + Only set vm.max_map_count if greater than default (#31512) + Add Get Snapshots High Level REST API (#31537) + QA: Merge query-builder-bwc to restart test (#30979) + Update reindex.asciidoc (#31626) + Docs: Skip xpack snippet tests if no xpack (#31619) + mute CreateSnapshotRequestTests + HLRest: Fix test for explain API + [TEST] Fix RemoteClusterConnectionTests + Add Create Snapshot to High-Level Rest Client (#31215) + Remove legacy MetaDataStateFormat (#31603) + Add explain API to high-level REST client (#31387) + Preserve thread context when connecting to remote cluster (#31574) + Unify headers for full text queries + Remove redundant 'minimum_should_match' + JDBC driver prepared statement set* methods (#31494) + [TEST] call yaml client close method from test suite (#31591) + +commit 0522c6644d3743e634c20487600964073a972292 +Author: Nik Everett +Date: Thu Jun 28 10:47:19 2018 -0400 + + Docs: Remove duplicate test setup + + The range docs had an introductory section that described how to set up + and index *and* a test setup section in `docs/build.gradle` that + duplicated that section. This is bad because these section can (and do) + drift from one another. This change removes the setup in build.gradle + and marks the introductor snippet with `// TESTSETUP` so it is used on + all the snippets. + +commit 304316ac3ba4c20837858b9d2eb069c504bd1747 +Author: Alpar Torok +Date: Thu Jun 28 17:58:24 2018 +0300 + + Print output when the name checker IT fails (#31660) + +commit e4315cc71060c130f9449c3c4f6614355f2db21d +Author: Nhat Nguyen +Date: Thu Jun 28 09:51:40 2018 -0400 + + Use external version type for history ops from Lucene (#31568) + + Today we use INTERNAL version type for the history operations from + Lucene index. However, this does not cover the case in which the + original operation has the version type EXTERNAL and version number 0. + Semantically all operations from translog or Lucene should always use + EXTERNAL version type. + +commit 3fce13fd28378e144c747ac0f72691e749cba48f +Author: Tim Brooks +Date: Thu Jun 28 07:40:33 2018 -0600 + + Fix syntax errors in get-snapshots docs (#31656) + + This is related to #31537. It fixes two syntax errors that are breaking + the docs build. + +commit 050fbc8f3d9d67cfaf355f9e047d332f1ed04285 +Author: Peter Evers +Date: Thu Jun 28 15:29:56 2018 +0200 + + Docs: Fix description of percentile ranks example example (#31652) + +commit 09dd19a40315d57d056b0014567018c2ceb5001b +Author: markharwood +Date: Thu Jun 28 14:05:26 2018 +0100 + + Add MultiSearchTemplate support to High Level Rest client (#30836) + + Add MultiSearchTemplate support to High Level Rest client. + Addresses part of #27205 + +commit 48cfb9b0db1afef80c4a6c2940d94cdcfa4cd2e9 +Author: Luca Cavanna +Date: Thu Jun 28 15:00:36 2018 +0200 + + Add test for low-level client round-robin behaviour (#31616) + +commit 0b1a0641efb306caf073a63d5683fe0deade3f58 +Author: Igor Motov +Date: Thu Jun 28 05:56:16 2018 -0700 + + SQL: Refactor package names of sql-proto and sql-shared-proto projects (#31622) + + - renames project `sql-proto` to `sql-action`, + - renames package `sql.plugin` to `sql.action` + - renames project `sql-shared-client` to `sql-client` + - renames package `sql.shard.client` to `sql.client` + - renames project `sql-shared-proto` to `sql-proto` + + Closes #31593 + +commit 0afec8f31c14ba1a6dfdd6fbeef118fa174721fa +Author: Alpar Torok +Date: Thu Jun 28 15:14:34 2018 +0300 + + Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389) + + * Remove deprecation warnings to prepare for Gradle 5 + + Gradle replaced `project.sourceSets.main.output.classesDir` of type + `File` with `project.sourceSets.main.output.classesDirs` of type + `FileCollection` + (see [SourceSetOutput](https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/java/org/gradle/api/tasks/SourceSetOutput.java)) + Build output is now stored on a per language folder. + + There are a few places where we use that, here's these and how it's + fixed: + + - Randomized Test execution + - look in all test folders ( pass the multi dir configuration to the + ant runner ) + - DRY the task configuration by introducing `basedOn` for + `RandomizedTestingTask` DSL + - Extend the naming convention test to support passing in multiple + directories + - Fix the standalon test plugin, the dires were not passed trough, + checked with a debuger and the statement had no affect due to a + missing `=`. + + Closes #30354 + + * Only check Java tests, PR feedback + + - Name checker was ran for Groovy tests that don't adhere to the same + convections causing the check to fail + - implement PR feedback + + * Replace `add` with `addAll` + + This worked because the list is passed to `project.files` that does the + right thing. + + * Revert "Only check Java tests, PR feedback" + + This reverts commit 9bd9389875d8b88aadb50df57a45cd0d2b073241. + + * Remove `basedOn` helper + + * Bring some changes back + + Previus revert accidentally reverted too much + + * Fix negation + + * add back public + + * revert name check changes + + * Revert "revert name check changes" + + This reverts commit a2800c0b363168339ea65e2a79ec8256e5883e6d. + + * Pass all dirs to name check + + Only run on Java for build-tools, this is safe because it's a self test. + It needs more work before we could pass in the Groovy classes as well as + these inherit from `GroovyTestCase` + + * remove self tests from name check + + The self complicates the task setup and disable real checks on + build-tools. + With this change there are no more self tests, and the build-tools tests + adhere to the conventions. + The self test will be replaced by gradle test kit, thus the addition of + the Gradle plugin builder plugin. + + * First test to run a Gradle build + + * Add tests that replace the name check self test + + * Clean up integ test base class + + * Always run tests + + * Align with test naming conventions + + * Make integ. test case inherit from unit test case + + The check requires this + + * Remove `import static org.junit.Assert.*` + +commit b1e05856354e3e6202239c706ec3ec67c381f698 +Author: Alpar Torok +Date: Thu Jun 28 15:02:05 2018 +0300 + + Correct integTest enable logic (#31646) + + Run xPack plugin integration tests when not on oss distro + +commit cff7be16fb95da062081a9c279aab7c6617927d6 +Author: Tim Brooks +Date: Thu Jun 28 05:04:04 2018 -0600 + + Fix missing get-snapshots docs reference #31645 + + This is related to #31537. It fixes a missing docs references in + get_snapshots.asciidoc. + +commit 0f0d13ee4c1c551c92a8a72663a7c050a9fcb570 +Author: Tanguy Leroux +Date: Thu Jun 28 10:57:49 2018 +0200 + + Do not check for Azure container existence (#31617) + + The current AzureStorageServiceImpl always checks if the Azure container + exists before reading or writing an object to the Azure container. This commit + removes this behavior, reducing the number of overhall requests executed + for all snapshots operations. + +commit 9d523d05ed016574aea68549a4bd11ffd1a9a01f +Author: Tanguy Leroux +Date: Thu Jun 28 10:40:53 2018 +0200 + + Merge AwsS3Service and InternalAwsS3Service in a S3Service class (#31580) + + The interface and its implementation can be merged into a single class, + which is renamed to S3Service like the other S3BlobStore, S3Repository + classes. + +commit 2b53510b4f264058be62a42c1420271281f53583 +Merge: b848ff27a60 8b698f0bce8 +Author: Colin Goodheart-Smithe +Date: Thu Jun 28 08:35:57 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 8557bbab28a52106536f4f47fbe3f36a7f3951be +Author: Alpar Torok +Date: Thu Jun 28 08:13:21 2018 +0300 + + Upgrade gradle wrapper to 4.8 (#31525) + + * Move to Gradle 4.8 RC1 + + * Use latest version of plugin + + The current does not work with Gradle 4.8 RC1 + + * Switch to Gradle GA + + * Add and configure build compare plugin + + * add work-around for https://github.com/gradle/gradle/issues/5692 + + * work around https://github.com/gradle/gradle/issues/5696 + + * Make use of Gradle build compare with reference project + + * Make the manifest more compare friendly + + * Clear the manifest in compare friendly mode + + * Remove animalsniffer from buildscript classpath + + * Fix javadoc errors + + * Fix doc issues + + * reference Gradle issues in comments + + * Conditionally configure build compare + + * Fix some more doclint issues + + * fix typo in build script + + * Add sanity check to make sure the test task was replaced + + Relates to #31324. It seems like Gradle has an inconsistent behavior and + the taks is not always replaced. + + * Include number of non conforming tasks in the exception. + + * No longer replace test task, create implicit instead + + Closes #31324. The issue has full context in comments. + + With this change the `test` task becomes nothing more than an alias for `utest`. + Some of the stand alone tests that had a `test` task now have `integTest`, and a + few of them that used to have `integTest` to run multiple tests now only + have `check`. + This will also help separarate unit/micro tests from integration tests. + + * Revert "No longer replace test task, create implicit instead" + + This reverts commit f1ebaf7d93e4a0a19e751109bf620477dc35023c. + + * Fix replacement of the test task + + Based on information from gradle/gradle#5730 replace the task taking + into account the task providres. + Closes #31324. + + * Only apply build comapare plugin if needed + + * Make sure test runs before integTest + + * Fix doclint aftter merge + + * PR review comments + + * Switch to Gradle 4.8.1 and remove workaround + + * PR review comments + + * Consolidate task ordering + +commit 909a18add77aa1027d7cb1620912136b678310f8 +Author: Ben Abrams +Date: Wed Jun 27 20:38:30 2018 -0700 + + Only set vm.max_map_count if greater than default (#31512) + + So the issue here is that we want to avoid setting vm.max_map_count if + it is already equal to the desired value (the bootstrap check requires + 262144). The reason we want to avoid this is because in some use-cases + using sysctl to set this will fail. In this case, we want to enable + users to set this value externally and then allow that to cause using + sysctl to set the value to be skipped so that cases where using sysctl + will fail to no longer fail. + +commit 9ac81a13220f3843e18aacd12f64ac4ab71447d8 +Author: Tim Brooks +Date: Wed Jun 27 18:04:34 2018 -0600 + + Add Get Snapshots High Level REST API (#31537) + + With this commit we add the get snapshots API to the Java high level + REST client. + + Relates #27205 + +commit d0c276c4562bbf3f8e9535396f7c5fdae853e5c5 +Author: Nik Everett +Date: Wed Jun 27 16:37:04 2018 -0400 + + QA: Merge query-builder-bwc to restart test (#30979) + + Merges the `query-builder-bwc` qa project into the + `full-cluster-restart` qa project, saving a cluster starts on every + build and *many* cluster starts on `./gradlew bwcTests`. + +commit 50e60a510dec544285a5542d618eb07a1c725c9e +Author: DeDe Morton +Date: Wed Jun 27 12:46:29 2018 -0700 + + Update reindex.asciidoc (#31626) + +commit b8907b04b1a18afa0f86aed18e103b3d59b12798 +Author: Nik Everett +Date: Wed Jun 27 15:41:44 2018 -0400 + + Docs: Skip xpack snippet tests if no xpack (#31619) + + Skips tests the require xpack if we run the doc build without xpack. So + this should work: + ``` + ./gradlew -p docs check -Dtests.distribution=oss-zip + ``` + + This is implemented by detecting parts of the doc that look like: + ``` + [testenv="basic"] + ``` + + Relates to #30665 + +commit a36819596517e6ef1019ebaba31c12149814b052 +Author: Vladimir Dolzhenko +Date: Wed Jun 27 21:36:28 2018 +0200 + + mute CreateSnapshotRequestTests + +commit ce1d0f5afdc505430e8a8bddb6545e5329ac8b1b +Author: Nik Everett +Date: Wed Jun 27 14:04:14 2018 -0400 + + HLRest: Fix test for explain API + + The test could sometimes generate an empty array of random stored fields + to fetch which would cause the test to fail. It is fine not to allow + empty here because we already randomize whether or not to generate the + stored fields param anyway. + +commit 7520025d7dbfa9f9ae7fec3dc50788824bf1fa36 +Author: Yannick Welsch +Date: Wed Jun 27 18:43:49 2018 +0200 + + [TEST] Fix RemoteClusterConnectionTests + + With PR #31574 we now ensure that connections are established under the proper + thread context. There is a test in RemoteClusterConnectionTests, however, + that shuts down the service while connecting. With the above change, a new kind + of exception can occur that the test is unaware of. + +commit 61eefc84f33d18636d0f559527ec4ab106ed52ab +Author: Jack Conradson +Date: Wed Jun 27 09:30:10 2018 -0700 + + Add Create Snapshot to High-Level Rest Client (#31215) + + Added support to the high-level rest client for the create snapshot API call. This required + several changes to toXContent which may need to be cleaned up in a later PR. Also + added several parsers for fromXContent to be able to retrieve appropriate responses + along with tests. + +commit 01623f66de772762cdaa4c4b2ee24c52da6d9d48 +Author: Yannick Welsch +Date: Wed Jun 27 17:36:58 2018 +0200 + + Remove legacy MetaDataStateFormat (#31603) + + Removes the legacy (pre-1.5) legacy MetaDataStateFormat. + +commit 616703b88063ec5f9176354562e4da2a5628a11a +Author: Yu +Date: Wed Jun 27 21:20:56 2018 +0800 + + Add explain API to high-level REST client (#31387) + + Relates to #27205 + +commit b7246199db2c788250e03b804493b3548df8b288 +Author: Yannick Welsch +Date: Wed Jun 27 10:46:42 2018 +0200 + + Preserve thread context when connecting to remote cluster (#31574) + + Establishing remote cluster connections uses a queue to coordinate multiple concurrent connect + attempts. Connect attempts can be initiated by user triggered searches as well as by system events + (e.g. when nodes disconnect). Multiple such concurrent events can lead to the connectListener of + one event to be called under the thread context of another connect attempt. This can lead to the + situation as seen in #31462 where the connect listener is executed under the system context, which + breaks when fetching the search shards from the remote cluster. + + Closes #31462 + +commit 4fc833b1decd71b5d3ded54d3a5ee2411bf34e00 +Author: Piotr Prądzyński +Date: Wed Jun 27 10:08:28 2018 +0200 + + Unify headers for full text queries + + Relates #31599 + +commit f6c64a048d87fd2d7dd66fab34ca471afc6906fe +Author: Piotr Prądzyński +Date: Wed Jun 27 10:08:55 2018 +0200 + + Remove redundant 'minimum_should_match' + + Relates #31600 + +commit 400db4f37df1b3806b7fa8295ddd18c67eb7f27d +Author: Andrei Stefan +Date: Wed Jun 27 10:05:39 2018 +0300 + + JDBC driver prepared statement set* methods (#31494) + + Added setObject functionality and tests for it + +commit a35b5341c41ef06022d4644be5d601654e16d77b +Author: Luca Cavanna +Date: Wed Jun 27 08:23:53 2018 +0200 + + [TEST] call yaml client close method from test suite (#31591) + + We added a way to close the yaml test client with #31575. + Such close method also needs to be called from the test suite though for + the additional clients to be closed. + +commit a55f614b8541852b3cf9535a0267767df59d45e0 +Merge: 3cd8392667e 13e1cf61917 +Author: Nhat Nguyen +Date: Tue Jun 26 16:25:32 2018 -0400 + + Merge branch 'master' into ccr + + * master: + ingest: Add ignore_missing property to foreach filter (#22147) (#31578) + Fix a formatting issue in the docvalue_fields documentation. (#31563) + reduce log level at gradle configuration time + [TEST] Close additional clients created while running yaml tests (#31575) + Docs: Clarify sensitive fields watcher encryption (#31551) + Watcher: Remove never executed code (#31135) + Add support for switching distribution for all integration tests (#30874) + Improve robustness of geo shape parser for malformed shapes (#31449) + QA: Create xpack yaml features (#31403) + Improve test times for tests using `RandomObjects::addFields` (#31556) + [Test] Add full cluster restart test for Rollup (#31533) + Enhance thread context uniqueness assertion + [DOCS] Fix heading format errors (#31483) + fix writeIndex evaluation for aliases (#31562) + Add x-opaque-id to search slow logs (#31539) + Watcher: Fix put watch action (#31524) + Add package pre-install check for java binary (#31343) + Reduce number of raw types warnings (#31523) + Migrate scripted metric aggregation scripts to ScriptContext design (#30111) + turn GetFieldMappingsResponse to ToXContentObject (#31544) + Close xcontent parsers (partial) (#31513) + Ingest Attachment: Upgrade Tika to 1.18 (#31252) + TEST: Correct the assertion arguments order (#31540) + +commit 3cd8392667e5c6625451f657376c9b40e6664ac1 +Author: Nhat Nguyen +Date: Tue Jun 26 16:23:41 2018 -0400 + + TEST: Skip asserting Lucene history for an empty index + + This maximization is not correct if the Lucene index is empty. + +commit 13e1cf61917285ee0f16d4c28818daeae6607525 +Author: Armin Braun +Date: Tue Jun 26 20:04:41 2018 +0200 + + ingest: Add ignore_missing property to foreach filter (#22147) (#31578) + +commit 26a927a120010c33273c845d0026f6c98e489012 +Author: Julie Tibshirani +Date: Tue Jun 26 10:15:56 2018 -0700 + + Fix a formatting issue in the docvalue_fields documentation. (#31563) + +commit db90905405297a392707c3137e19f7227462bdd8 +Author: Alpar Torok +Date: Tue Jun 26 18:16:24 2018 +0300 + + reduce log level at gradle configuration time + +commit 823a9d34dafabcfdfc9b1ee7f87f61a9db4f7b1b +Author: Luca Cavanna +Date: Tue Jun 26 16:56:35 2018 +0200 + + [TEST] Close additional clients created while running yaml tests (#31575) + + We recently introduced a mechanism that allows to specify a node + selector as part of do sections (see #31471). When a node selector that + is not the default one is configured, a new client will be initialized + with the same properties as the default one, but with the specified + node selector. This commit improves such mechanism but also closing + the additional clients being created and adding equals/hashcode impl to + the custom node selector as they are cached into a map. + +commit 8a6d0621804c98e61176fd771923dcbfeac8aaa0 +Author: Alexander Reelsen +Date: Tue Jun 26 16:24:28 2018 +0200 + + Docs: Clarify sensitive fields watcher encryption (#31551) + + Clarify the scope of encrypting sensitive settings in watcher, which fields + are encrypted and if users can have their own encrypted fields. + +commit a72dc9e8fc4eeb174ef3d6a20738523b70c2e212 +Author: Alexander Reelsen +Date: Tue Jun 26 16:14:40 2018 +0200 + + Watcher: Remove never executed code (#31135) + + The removed code snippet was never executed, as the version was never set and + thus always -1, after parsing the watch. With the changes done in + c9d77d20fdb8a46b614515555d5d0697b2c6e5fc this logic would not have + worked correctly anyway. + +commit 08b8d11e303e6d949d7999b0470cb0af835ed283 +Author: Alpar Torok +Date: Tue Jun 26 06:49:03 2018 -0700 + + Add support for switching distribution for all integration tests (#30874) + + * remove left-over comment + + * make sure of the property for plugins + + * skip installing modules if these exist in the distribution + + * Log the distrbution being ran + + * Don't allow running with integ-tests-zip passed externally + + * top level x-pack/qa can't run with oss distro + + * Add support for matching objects in lists + + Makes it possible to have a key that points to a list and assert that a + certain object is present in the list. All keys have to be present and + values have to match. The objects in the source list may have additional + fields. + + example: + ``` + match: { 'nodes.$master.plugins': { name: ingest-attachment } } + ``` + + * Update plugin and module tests to work with other distributions + + Some of the tests expected that the integration tests will always be ran + with the `integ-test-zip` distribution so that there will be no other + plugins loaded. + + With this change, we check for the presence of the plugin without + assuming exclusivity. + + * Allow modules to run on other distros as well + + To match the behavior of tets.distributions + + * Add and use a new `contains` assertion + + Replaces the previus changes that caused `match` to do a partial match. + + * Implement PR review comments + +commit 544e473908c10674446b31edc4027a91c29ff82e +Author: Igor Motov +Date: Tue Jun 26 06:44:10 2018 -0700 + + Improve robustness of geo shape parser for malformed shapes (#31449) + + Ensures that malformed geoshapes are more reliably ignored if + "ignore_malformed" is set to true instead of failing the entire + document. + + Closes #31428 + +commit 232c71b6bf22bdddee6faa8d46b5a96029b03942 +Author: Nik Everett +Date: Tue Jun 26 09:26:48 2018 -0400 + + QA: Create xpack yaml features (#31403) + + This creates a YAML test "features" that indices if the cluster being + tested has xpack installed (`xpack`) or if it does *not* have xpack + installed (`no_xpack`). It uses those features to centralize skipping + a few tests that fail if xpack is installed. + + The plan is to use this in a followup to skip docs tests that require + xpack when xpack is not installed. We *plan* to use the declaration + of required license level on the docs page to generate the required + `skip`. + + Closes #30933. + +commit ca4c857a9022b0db9dc8a10465224d1e9037a433 +Author: Sohaib Iftikhar +Date: Tue Jun 26 12:39:53 2018 +0200 + + Improve test times for tests using `RandomObjects::addFields` (#31556) + + Currently RandomObjects::addFields can potentially generate a large number of fields This commit decreases the chances that a new object or array is added as a new branch of an object, which lowers the probability of ending up with very big documents generated. It also reduces the number of documents generated for the SimulatePipelineResponseTests from 10 to 5 to reduce the testing time required for parsing. + +commit be9292cac6a4741af678dbf885daebc963977336 +Author: Tanguy Leroux +Date: Tue Jun 26 10:07:25 2018 +0200 + + [Test] Add full cluster restart test for Rollup (#31533) + + This pull request adds a full cluster restart test for a Rollup job. + The test creates and starts a Rollup job on the cluster and checks + that the job already exists and is correctly started on the upgraded + cluster. + + This test allows to test that the persistent task state is correctly + parsed from the cluster state after the upgrade, as the status field + has been renamed to state in #31031. + + The test undercovers a ClassCastException that can be thrown in + the RollupIndexer when the timestamp as a very low value that fits + into an integer. When it's the case, the value is parsed back as an + Integer instead of Long object and (long) position.get(rollupFieldName) + fails. + +commit 424be027283588ac5fb76dffbee1c1cb88e75ff5 +Author: Jason Tedor +Date: Tue Jun 26 00:01:45 2018 -0400 + + Enhance thread context uniqueness assertion + + This commit enhances the assertion message when de-duplicating values on + a thread context header so that if this assertion trips we can see the + values and their correpsonding unique values to understand why this + assertion might trip. + +commit 357a07e7a21fefa6144de6829e1e2042a2960786 +Author: Sue Gallagher <36747279+Sue-Gallagher@users.noreply.github.com> +Date: Mon Jun 25 17:25:32 2018 -0700 + + [DOCS] Fix heading format errors (#31483) + + * [DOCS] Fix heading format errors. Closes #31327 + + * [DOCS] Fix heading format errors. Closes #31327 + +commit c6012ed5753dfa9475a2e3be6b31a33253c5bb4f +Author: Tal Levy +Date: Mon Jun 25 17:17:16 2018 -0700 + + fix writeIndex evaluation for aliases (#31562) + + AliasOrIndex.Alias#writeIndex was returning a write index when + an alias was pointing to only one index, regardless whether `is_write_index` was + set to `false`. This fixes that so that there is no write index in such a case + that an alias points to only one index with `is_write_index=false`. + +commit 237650e9c054149fd08213b38a81a3666c1868e5 +Author: Igor Motov +Date: Mon Jun 25 12:20:27 2018 -0700 + + Add x-opaque-id to search slow logs (#31539) + + Add x-opaque-id to search slow logs only. Indexing slow log and audit + logs will be handled as separate PRs. + + Relates #31521 + +commit bb1d4aaf176a80818477f07072ccaa15c29fefc2 +Author: Alexander Reelsen +Date: Mon Jun 25 18:25:34 2018 +0200 + + Watcher: Fix put watch action (#31524) + + If no version is specified when putting a watch, the index API should be + used instead of the update API, so that the whole watch gets overwritten + instead of being merged with the existing one. + + Merging only happens when a version is specified, so that credentials can be omitted, which is important for the watcher UI. + +commit b848ff27a6090d1abdd30eddca717c501736f8d8 +Author: Colin Goodheart-Smithe +Date: Mon Jun 25 17:18:50 2018 +0100 + + Renames endpoints for setting and deleting ILM policies# (#31552) + +commit adfcea2af6c5eaaf4160d223664bea138b41095f +Author: Michael Basnight +Date: Mon Jun 25 10:54:39 2018 -0500 + + Add package pre-install check for java binary (#31343) + + The package installation relies on java being in the path. If java is + not in the path, the tests fail at post-install time. This commit adds a + pre-install check to validate that java exists, and if it fails, the + package is never installed, and thus keeps a system clean, rather than + aborting at post-install and leaving behind a mess. + + Closes #29665 + +commit 86ab3a2d1a3bc5f4478d25cd28906915c2ec03fd +Author: Christoph Büscher +Date: Mon Jun 25 15:59:03 2018 +0200 + + Reduce number of raw types warnings (#31523) + + A first attempt to reduce the number of raw type warnings, + most of the time by using the unbounded wildcard. + +commit 8e4768890abbd2fb6fbc49d6188909ed7ca6961b +Author: Jonathan Little +Date: Mon Jun 25 04:01:33 2018 -0700 + + Migrate scripted metric aggregation scripts to ScriptContext design (#30111) + + * Migrate scripted metric aggregation scripts to ScriptContext design #29328 + + * Rename new script context container class and add clarifying comments to remaining references to params._agg(s) + + * Misc cleanup: make mock metric agg script inner classes static + + * Move _score to an accessor rather than an arg for scripted metric agg scripts + + This causes the score to be evaluated only when it's used. + + * Documentation changes for params._agg -> agg + + * Migration doc addition for scripted metric aggs _agg object change + + * Rename "agg" Scripted Metric Aggregation script context variable to "state" + + * Rename a private base class from ...Agg to ...State that I missed in my last commit + + * Clean up imports after merge + +commit 6be94dbecdded9796203b05cc95162a00b1a3dd1 +Author: Vladimir Dolzhenko +Date: Mon Jun 25 10:12:31 2018 +0200 + + turn GetFieldMappingsResponse to ToXContentObject (#31544) + +commit e6dc01d2b7d36b6d2a6aa11289afb70c9568c986 +Author: Tom Callahan +Date: Sun Jun 24 15:39:56 2018 -0400 + + Close xcontent parsers (partial) (#31513) + + Partial pass at closing XContentParsers in server. This mostly involved adding try-with-resources statements around the usage of XContentParsers. + +commit 0f956290eae510a708c14e3ad8ac3034eb3455c8 +Author: Jack Conradson +Date: Sun Jun 24 11:08:45 2018 -0700 + + Ingest Attachment: Upgrade Tika to 1.18 (#31252) + + Fixes ES from hanging when a bad zip file is loaded through Tika. + +commit 43c2e2fed14b8866733fcb41ea1e86dd185842e0 +Author: olcbean +Date: Sun Jun 24 04:14:34 2018 +0200 + + TEST: Correct the assertion arguments order (#31540) + + This commit corrects the order of the assertion arguments in HLRest client. + +commit 27bd68862ad2be45dce836a088c607124407eb0a +Author: Vladimir Dolzhenko +Date: Sat Jun 23 09:39:17 2018 +0200 + + Add get field mappings to High Level REST API Client (#31423) + + Add get field mappings to High Level REST API Client + Relates to #27205 + +commit 54d1b9eb6fc4cf88bc7b41b469dba8f6b0b21310 +Author: Lisa Cawley +Date: Fri Jun 22 18:09:37 2018 -0700 + + [DOCS] Updates Watcher examples for code testing (#31152) + +commit 44e1c8e9a96c875aac68c4e7c164fdb7ddfb6855 +Author: Nhat Nguyen +Date: Fri Jun 22 17:42:58 2018 -0400 + + TEST: Add bwc recovery tests with synced-flush index + + Although the master branch does not affect by #31482, it's helpful to + have BWC tests that verify the peer recovery with a synced-flush index. + This commit adds the bwc tests from #31506 to the master branch. + + Relates #31482 + Relates #31506 + +commit efffa0226de6081bf1e253f88454e6c5ca59770d +Author: Lisa Cawley +Date: Fri Jun 22 15:40:25 2018 -0700 + + [DOCS] Move sql to docs (#31474) + +commit baa9b8575c911f274ac0feba911588b4ff5b8bb0 +Author: Lisa Cawley +Date: Fri Jun 22 15:39:34 2018 -0700 + + [DOCS] Move monitoring to docs folder (#31477) + +commit 49387bf2d743850e257c649432af0b9ef198a34a +Author: Ryan Ernst +Date: Fri Jun 22 15:03:01 2018 -0700 + + Core: Combine doExecute methods in TransportAction (#31517) + + TransportAction currently contains 2 doExecute methods, one which takes + a the task, and one that does not. The latter is what some subclasses + implement, while the first one just calls the latter, dropping the given + task. This commit combines these methods, in favor of just always + assuming a task is present. + +commit 2ba50dbf50b273d420c21dcdb923fd163bfab182 +Author: Vladimir Dolzhenko +Date: Fri Jun 22 21:08:11 2018 +0200 + + IndexShard should not return null stats (#31528) + + IndexShard should not return null stats - empty stats or AlreadyCloseException if it's closed is better + +commit 8b698f0bce8c73cb6d8510ce6a3f857cb0fb2f02 +Author: Vladimir Dolzhenko +Date: Mon Jun 25 10:12:31 2018 +0200 + + turn GetFieldMappingsResponse to ToXContentObject (#31544) + +commit 629c376f02440fdbc704645c9568327044b939dc +Author: Tom Callahan +Date: Sun Jun 24 15:39:56 2018 -0400 + + Close xcontent parsers (partial) (#31513) + + Partial pass at closing XContentParsers in server. This mostly involved adding try-with-resources statements around the usage of XContentParsers. + +commit 9efb0fe9bafef8c681e39cc143e864be61949f8f +Author: Jack Conradson +Date: Sun Jun 24 11:08:45 2018 -0700 + + Ingest Attachment: Upgrade Tika to 1.18 (#31252) + + Fixes ES from hanging when a bad zip file is loaded through Tika. + +commit e401710f6ea7fd43b1ddb70dd3f5a017d90a1eaf +Author: olcbean +Date: Sun Jun 24 04:14:34 2018 +0200 + + TEST: Correct the assertion arguments order (#31540) + + This commit corrects the order of the assertion arguments in HLRest client. + +commit 2c56df631d1d4127cc5ed0c6e6d34f865f7a4866 +Author: Nhat Nguyen +Date: Sat Jun 23 17:53:18 2018 -0400 + + Adjusts transport actions in CCR + + This commit adjusts the ccr’s actions accordingly to the recent changes + in the upstream. + +commit 08ee9b67c5c7bc941f93bc3703748ee889ce7e7d +Merge: 601ea76b76d b7ef75fed6d +Author: Nhat Nguyen +Date: Sat Jun 23 17:27:17 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add get field mappings to High Level REST API Client (#31423) + [DOCS] Updates Watcher examples for code testing (#31152) + TEST: Add bwc recovery tests with synced-flush index + [DOCS] Move sql to docs (#31474) + [DOCS] Move monitoring to docs folder (#31477) + Core: Combine doExecute methods in TransportAction (#31517) + IndexShard should not return null stats (#31528) + fix repository update with the same settings but different type (#31458) + Fix Mockito trying to mock IOException that isn't thrown by method (#31433) (#31527) + Node selector per client rather than per request (#31471) + Core: Combine messageRecieved methods in TransportRequestHandler (#31519) + Upgrade to Lucene 7.4.0. (#31529) + [ML] Add ML filter update API (#31437) + Allow multiple unicast host providers (#31509) + Avoid deprecation warning when running the ML datafeed extractor. (#31463) + REST high-level client: add simulate pipeline API (#31158) + Get Mapping API to honour allow_no_indices and ignore_unavailable (#31507) + [PkiRealm] Invalidate cache on role mappings change (#31510) + [Security] Check auth scheme case insensitively (#31490) + In NumberFieldType equals and hashCode, make sure that NumberType is taken into account. (#31514) + [DOCS] Fix REST tests in SQL docs + [DOCS] Add code snippet testing in more ML APIs (#31339) + Core: Remove ThreadPool from base TransportAction (#31492) + [DOCS] Remove fixed file from build.gradle + Rename createNewTranslog to fileBasedRecovery (#31508) + Test: Skip assertion on windows + [DOCS] Creates field and document level security overview (#30937) + [DOCS] Significantly improve SQL docs + [DOCS] Move migration APIs to docs (#31473) + Core: Convert TransportAction.execute uses to client calls (#31487) + Return transport addresses from UnicastHostsProvider (#31426) + Ensure local addresses aren't null (#31440) + Remove unused generic type for client execute method (#31444) + Introduce http and tcp server channels (#31446) + +commit b7ef75fed6d75e1a88b000e36ee78a018c0c53d0 +Author: Vladimir Dolzhenko +Date: Sat Jun 23 09:39:17 2018 +0200 + + Add get field mappings to High Level REST API Client (#31423) + + Add get field mappings to High Level REST API Client + Relates to #27205 + +commit b6cc6fc2bc44fbde69617566d549829c466cccda +Author: Lisa Cawley +Date: Fri Jun 22 18:09:37 2018 -0700 + + [DOCS] Updates Watcher examples for code testing (#31152) + +commit 51151027cdee34d668802b90dba263b2f4bc8695 +Author: Nhat Nguyen +Date: Fri Jun 22 17:42:58 2018 -0400 + + TEST: Add bwc recovery tests with synced-flush index + + Although the master branch does not affect by #31482, it's helpful to + have BWC tests that verify the peer recovery with a synced-flush index. + This commit adds the bwc tests from #31506 to the master branch. + + Relates #31482 + Relates #31506 + +commit 638b9fd88cdbe717d2e13bf4dbbda9ce610bc198 +Author: Lisa Cawley +Date: Fri Jun 22 15:40:25 2018 -0700 + + [DOCS] Move sql to docs (#31474) + +commit eb81a305ae36beed75dff7e61b05ec10c69f339b +Author: Lisa Cawley +Date: Fri Jun 22 15:39:34 2018 -0700 + + [DOCS] Move monitoring to docs folder (#31477) + +commit 7a150ec06d5b846caa89520c1388e9c751a0c8af +Author: Ryan Ernst +Date: Fri Jun 22 15:03:01 2018 -0700 + + Core: Combine doExecute methods in TransportAction (#31517) + + TransportAction currently contains 2 doExecute methods, one which takes + a the task, and one that does not. The latter is what some subclasses + implement, while the first one just calls the latter, dropping the given + task. This commit combines these methods, in favor of just always + assuming a task is present. + +commit f04c579203299bbbb9397609344b59e6f6f8f18f +Author: Vladimir Dolzhenko +Date: Fri Jun 22 21:08:11 2018 +0200 + + IndexShard should not return null stats (#31528) + + IndexShard should not return null stats - empty stats or AlreadyCloseException if it's closed is better + +commit 1d11fdaad80d19be782b99e3dfede28b40d8f0d1 +Merge: e6b427c8e4a 7313a987f4a +Author: Tal Levy +Date: Fri Jun 22 10:11:15 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 7313a987f4a8ba1e39d6105f7d74be9186faa95b +Author: Vladimir Dolzhenko +Date: Fri Jun 22 17:44:13 2018 +0200 + + fix repository update with the same settings but different type (#31458) + + fix repository update with the same settings but different type + +commit 3c42bfad4e68c464ee57420f39e390efd2888761 +Author: Armin Braun +Date: Fri Jun 22 17:24:27 2018 +0200 + + Fix Mockito trying to mock IOException that isn't thrown by method (#31433) (#31527) + +commit 16e4e7a7cfb5196e02e6fc988f34553dc9d34acc +Author: Luca Cavanna +Date: Fri Jun 22 17:15:29 2018 +0200 + + Node selector per client rather than per request (#31471) + + We have made node selectors configurable per request, but all + of other language clients don't allow for that. + A good reason not to do so, is that having a different node selector + per request breaks round-robin. This commit makes NodeSelector + configurable only at client initialization. It also improves the docs + on this matter, important given that a single node selector can still + affect round-robin. + +commit 59e7c6411a04f08a325f02d612e12eab12b22316 +Author: Ryan Ernst +Date: Fri Jun 22 07:36:03 2018 -0700 + + Core: Combine messageRecieved methods in TransportRequestHandler (#31519) + + TransportRequestHandler currently contains 2 messageReceived methods, + one which takes a Task, and one that does not. The first just delegates + to the second. This commit changes all existing implementors of + TransportRequestHandler to implement the version which takes Task, thus + allowing the class to be a functional interface, and eliminating the + need to throw exceptions when a task needs to be ensured. + +commit f023e95ae0ce4828085631b342d518c0e572a8aa +Author: Adrien Grand +Date: Fri Jun 22 16:17:17 2018 +0200 + + Upgrade to Lucene 7.4.0. (#31529) + + This moves Elasticsearch from a recent 7.4.0 snapshot to the GA release. + +commit c6cbc99f9c90c75e883393dc6b4691033ab73d72 +Author: Dimitris Athanasiou +Date: Fri Jun 22 15:13:31 2018 +0100 + + [ML] Add ML filter update API (#31437) + + This adds an api to allow updating a filter: + + POST _xpack/ml/filters/{filter_id}/_update + + The request body may have: + + - description: setting a new description + - add_items: a list of the items to add + - remove_items: a list of the items to remove + + This commit also changes the PUT filter api to + error when the filter_id is already used. As + now there is an api for updating filters, the + put api should only be used to create new ones. + + Also, updating a filter results into a notification + message auditing the change for every job that is + using that filter. + +commit f22f91c57a6199ddadb19b2bf839d3ac7c3e2fbd +Author: Yannick Welsch +Date: Fri Jun 22 15:31:23 2018 +0200 + + Allow multiple unicast host providers (#31509) + + Introduces support for multiple host providers, which allows the settings based hosts resolver to be + treated just as any other UnicastHostsProvider. Also introduces the notion of a HostsResolver so + that plugins such as FileBasedDiscovery do not need to create their own thread pool for resolving + hosts, making it easier to add new similar kind of plugins. + +commit 8ae2049889766e4c8cbe67bd3f0d1d9998c542a5 +Author: Adrien Grand +Date: Fri Jun 22 13:46:48 2018 +0200 + + Avoid deprecation warning when running the ML datafeed extractor. (#31463) + + In #29639 we added a `format` option to doc-value fields and deprecated usage + of doc-value fields without a format so that we could migrate doc-value fields + to use the format that comes with the mappings by default. However I missed to + fix the machine-learning datafeed extractor. + +commit e6b427c8e4a3a44861286ff1ed37234efec8bad0 +Author: Colin Goodheart-Smithe +Date: Fri Jun 22 10:03:43 2018 +0100 + + Fixes compile error due to client interface change + +commit d0625567540acd43aecbeee7d303740687468c67 +Merge: c9eec7eadcb eade1618940 +Author: Colin Goodheart-Smithe +Date: Fri Jun 22 10:00:35 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit eade161894011cf19d47d14e4c3c1e15dd945227 +Author: Sohaib Iftikhar +Date: Fri Jun 22 09:59:04 2018 +0200 + + REST high-level client: add simulate pipeline API (#31158) + + relates to #27205 + +commit 0352d88621c1d3794197105f7fef97e669c40179 +Author: Luca Cavanna +Date: Fri Jun 22 09:57:32 2018 +0200 + + Get Mapping API to honour allow_no_indices and ignore_unavailable (#31507) + + Get Mapping currently throws index not found exception (and returns + 404 status code) from the REST layer whenever an index was specified + and no indices have been returned. We should not have this logic in the + REST layer though as only our index resolver should decide whether we + need to throw exceptions or not based on provided indices and corresponding + indices options. + + Closes #31485 + +commit 009ae48cbae967d102b28d8b8e0bf25197c90421 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Jun 22 17:47:20 2018 +1000 + + [PkiRealm] Invalidate cache on role mappings change (#31510) + + PkiRealm caches successful authentications and provides ways to + invalidate the cache. But in some scenario's the cache was not being + invalidated on role mapping change. + PkiRealm does not inform role mapper to be notified for cache + refresh on role mapping updates. + The logic in `TransportClearRealmCacheAction#nodeOperation` + which gets invoked for refreshing cache on realms, considers null or + empty realm names in the request as clear cache on all realms. When + LDAP realm is not present then it clears cache for all realms so it + works fine, but when LDAP realm is configured then role mapper + sends a request with LDAP realm names and so the cache is cleared + only for those realms. + + This commit resolves the issue by registering PkiRealm with role + mapper for cache refresh. PkiRealm implements CachingRealm and as it + does not extend CachingUsernamePasswordRealm, have modified the + interface method `refreshRealmOnChange` to accept CachingRealm. + +commit 724438a0b084a9a9d20790ce141bf8ec18265950 +Author: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> +Date: Fri Jun 22 10:15:38 2018 +1000 + + [Security] Check auth scheme case insensitively (#31490) + + According to RFC 7617, the Basic authentication scheme name + should not be case sensitive. + Case insensitive comparisons are also applicable for the bearer + tokens where Bearer authentication scheme is used as per + RFC 6750 and RFC 7235 + + Some Http clients may send authentication scheme names in + different case types for eg. Basic, basic, BASIC, BEARER etc., + so the lack of case-insensitive check is an issue when these + clients try to authenticate with elasticsearch. + + This commit adds case-insensitive checks for Basic and Bearer + authentication schemes. + + Closes #31486 + +commit 3b7225e9d1dead1718132a51fdde5d1cbe4a42b9 +Author: Julie Tibshirani +Date: Thu Jun 21 16:53:20 2018 -0700 + + In NumberFieldType equals and hashCode, make sure that NumberType is taken into account. (#31514) + +commit 99f503e3bea5c289907ff16b438875ecf2a8f2a2 +Author: Costin Leau +Date: Fri Jun 22 00:16:03 2018 +0300 + + [DOCS] Fix REST tests in SQL docs + + Fixed a search & replace gone awry + Tweaked the docs a bit + +commit 438591566f7de2a589ac70c14aa8f893f4ba5f16 +Author: Lisa Cawley +Date: Thu Jun 21 11:32:11 2018 -0700 + + [DOCS] Add code snippet testing in more ML APIs (#31339) + +commit 4f9332ee16bcc422144a866122eb8bc2df2c0040 +Author: Ryan Ernst +Date: Thu Jun 21 11:25:26 2018 -0700 + + Core: Remove ThreadPool from base TransportAction (#31492) + + Most transport actions don't need the node ThreadPool. This commit + removes the ThreadPool as a super constructor parameter for + TransportAction. The actions that do need the thread pool then have a + member added to keep it from their own constructor. + +commit 60204af0cbcf43c83e86b4ddc8426111139f4838 +Author: lcawl +Date: Thu Jun 21 11:13:19 2018 -0700 + + [DOCS] Remove fixed file from build.gradle + +commit 048a92bf39beecef1fa33e820699d79bdcd9133b +Author: Nhat Nguyen +Date: Thu Jun 21 13:50:46 2018 -0400 + + Rename createNewTranslog to fileBasedRecovery (#31508) + + We renamed `createNewTranslog` to `fileBasedRecovery` in the + RecoveryTarget but did not do this for RecoverySourceHandler. + This commit makes sure that we a consistent parameter in both + recovery source and target. + +commit 6f3e97f2b7536e9d092ba80cb71d4ff3f7557734 +Author: Nik Everett +Date: Thu Jun 21 13:24:39 2018 -0400 + + Test: Skip assertion on windows + + Windows doesn't provide consistent exception messages when it can't + connect so skip the exception message assertion on windows. + + Closes #31457 + +commit bd06563e78688fa3a94a8b17285a0a6c7565fbb9 +Author: Lisa Cawley +Date: Thu Jun 21 10:08:50 2018 -0700 + + [DOCS] Creates field and document level security overview (#30937) + +commit 872418ff9499011bf6ee8da473fc2c49bbc67a43 +Author: Costin Leau +Date: Wed Jun 20 21:40:13 2018 +0300 + + [DOCS] Significantly improve SQL docs + + Introduce SQL commands + Move reserved keywords into an appendix + Add section on security + Introduce concepts section + +commit c9eec7eadcbb7f233a8c09211f02f937f501371e +Merge: 901aafb946d 68ec9588737 +Author: Tal Levy +Date: Thu Jun 21 09:56:45 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 901aafb946d3b4dd22fff1194543ead43b48587e +Author: Tal Levy +Date: Thu Jun 21 09:56:02 2018 -0700 + + add APIs to Maintenance Mode in ILM (#31410) + + - POST _xpack/index_lifecycle/_stop + - issues a request to be placed into STOPPED mode (maintenance mode). + This is not immediate, since we must first verify that + it is safe to go from STOPPING -> STOPPED. + - POST _xpack/index_lifecycle/_start + - issues a request to be placed back into RUNNING mode (immediately) + - GET _xpack/index_lifecycle/_status + - get back the current mode our lifecycle management is in + + - update task was hardened to support uninstalled metadata + - if no metadata is installed, the start/stop actions will install metadata + and proceed to try and change it (default start mode is RUNNING) + - rename MAINTENANCE -> STOPPED, MAINTENANCE_REQUESTED -> STOPPING, NORMAL -> RUNNING + + follow-up to #31164. + +commit 68ec9588737d97c2a9282308fce10ac6f3cf03c7 +Author: Lisa Cawley +Date: Thu Jun 21 08:19:23 2018 -0700 + + [DOCS] Move migration APIs to docs (#31473) + +commit 0a324b9943758bb976f93c99031ce517507348f0 +Author: Ryan Ernst +Date: Thu Jun 21 07:59:55 2018 -0700 + + Core: Convert TransportAction.execute uses to client calls (#31487) + + This commit converts some of the existing calls to + TransportAction.execute to use the equivalent client method for the + desired action. + +commit fc2f982542a3a265a6ff145885ba28303ff22f10 +Author: Tal Levy +Date: Thu Jun 21 07:18:14 2018 -0700 + + inherit [index.lifecycle.date] from rolled-over time (#30853) + + Indices that are rolled over preserve their origin index.lifecycle.date + as the time the index was created. Although this is also fine, it would + make more sense to start the timer for moving to the warm phase from the + time it was rolled over so that we capture the time elapsed from the + latest data, as opposed to when the index was created. + + This commit adds an extra step within the RolloverAction that + extracts the index.creation_date of the newly created index from the RolloverStep + and sets that as the index.lifecycle.date of the rolled-over index that is + waiting for its next phase. + +commit da69ab28c70bb2817da1efcdd45a14781f741157 +Author: Yannick Welsch +Date: Thu Jun 21 16:00:26 2018 +0200 + + Return transport addresses from UnicastHostsProvider (#31426) + + With #20695 we removed local transport and there is just TransportAddress now. The + UnicastHostsProvider currently returns DiscoveryNode instances, where, during pinging, we're + actually only making use of the TransportAddress to establish a first connection to the possible new + node. To simplify the interface, we can just return a list of transport addresses instead, which + means that it's not necessary anymore to create fake node objects in each plugin just to return the + address information. + +commit 601ea76b76d6abac39898b5826bada8336748e95 +Author: Nhat Nguyen +Date: Thu Jun 21 09:30:08 2018 -0400 + + Use Lucene soft-deletes in peer recovery (#30522) + + This commit adds Lucene soft-deletes as another source for peer-recovery + besides translog. + + Relates #29530 + +commit 31976060d9c42e64a6197c5c695400b6a3d15514 +Author: Colin Goodheart-Smithe +Date: Thu Jun 21 12:18:59 2018 +0100 + + Adds ability to update a policy (#31361) + + * Adds ability to update a policy as long as no indexes are in the shrink + action + + * Address review comments + +commit dec6d49bd2a612888e0d3038c763182335c03c49 +Author: Colin Goodheart-Smithe +Date: Thu Jun 21 09:18:06 2018 +0100 + + Adds an API to remove ILM from an index completely (#31358) + + * Adds an API to remove ILM from an index completely + + The removal will only happen if the index is not in the shrink action + + * Fixes compile issues + +commit 86423f9563dd41cbf2a15b16962e2878ffe98185 +Author: Tim Brooks +Date: Wed Jun 20 19:50:14 2018 -0600 + + Ensure local addresses aren't null (#31440) + + Currently we set local addresses on the creation time of a NioChannel. + However, this may return null as the local address may not have been + set yet. An example is the local address has not been set on a client + channel as the connection process is not yet complete. + + This PR modifies the getter to set the local field if it is currently null. + +commit d467be300ed8d48d189a29b7d59931a5e048892e +Author: Nhat Nguyen +Date: Wed Jun 20 20:36:37 2018 -0400 + + Docs: Add ccr to the cat thread pool doc test (#31442) + + Since #31251, we use the default distribution to test docs. With this + distribution, the cat thread-pool response will include the ccr + thread-pool. This commit adjusts /_cat/thread_pool doc to include ccr. + + Relates #31251 + +commit 00283a61e1dbcd6990f0ed7d369261a31fc89d8c +Author: Ryan Ernst +Date: Wed Jun 20 16:26:26 2018 -0700 + + Remove unused generic type for client execute method (#31444) + + This commit removes the request builder generic type for AbstractClient + as it was unused. + +commit 9ab1325953b30fb3d881e9c27b2afdd3274322b6 +Author: Tim Brooks +Date: Wed Jun 20 16:34:56 2018 -0600 + + Introduce http and tcp server channels (#31446) + + Historically in TcpTransport server channels were represented by the + same channel interface as socket channels. This was necessary as + TcpTransport was parameterized by the channel type. This commit + introduces TcpServerChannel and HttpServerChannel classes. Additionally, + it adds the implementations for the various transports. This allows + server channels to have unique functionality and not implement the + methods they do not support (such as send and getRemoteAddress). + + Additionally, with the introduction of HttpServerChannel this commit + extracts some of the storing and closing channel work to the abstract + http server transport. + +commit 4c55dbce1abaf695d1a4b40f87e9b6369b4950e4 +Merge: 8d9b98569f9 8bfb9aadd9a +Author: Tal Levy +Date: Wed Jun 20 13:29:47 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 9f423daed69015c36707421dc821ec95e939edaa +Merge: 34f127be3c0 8bfb9aadd9a +Author: Nhat Nguyen +Date: Wed Jun 20 16:16:36 2018 -0400 + + Merge branch 'master' into ccr + +commit 8bfb9aadd9ae277d72f9b683af781e4231130184 +Author: Tal Levy +Date: Wed Jun 20 13:00:36 2018 -0700 + + lower rollover-info version bound to 6.4 (#31414) + + This feature is supported in 6.4, this commit + reflects that + +commit 4f850273b363aa6e7e93f778ae30038e34d70abb +Author: Tal Levy +Date: Wed Jun 20 13:00:18 2018 -0700 + + extend is-write-index serialization support to 6.4 (#31415) + +commit e2029232256298d89022f5314ba8d174cf43a168 +Author: Costin Leau +Date: Wed Jun 20 21:42:15 2018 +0300 + + [DOCS] Fix JDBC Maven client group/artifact ID + +commit 34f127be3c0edab31e479b3fb488bddd4a2ad871 +Author: Nhat Nguyen +Date: Wed Jun 20 13:20:24 2018 -0400 + + CCR: Remove index name resolver from CCR actions + + Relates #31002 + +commit efcb9a3603109ec22ff99b81fa43fecc69e18368 +Merge: c74cd30ac6f 6ebe6e32cf3 +Author: Nhat Nguyen +Date: Wed Jun 20 12:43:40 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [DOCS] Omit shard failures assertion for incompatible responses (#31430) + [DOCS] Move licensing APIs to docs (#31445) + Add Delete Snapshot High Level REST API + Remove QueryCachingPolicy#ALWAYS_CACHE (#31451) + [Docs] Extend Homebrew installation instructions (#28902) + Choose JVM options ergonomically + [Docs] Mention ip_range datatypes on ip type page (#31416) + Multiplexing token filter (#31208) + Fix use of time zone in date_histogram rewrite (#31407) + Core: Remove index name resolver from base TransportAction (#31002) + [DOCS] Fixes code snippet testing for machine learning (#31189) + [DOCS] Removed and params from MLT. Closes #28128 (#31370) + Security: fix joining cluster with production license (#31341) + Unify http channels and exception handling (#31379) + [DOCS] Moves the info API to docs (#31121) + Preserve response headers on cluster update task (#31421) + [DOCS] Add code snippet testing for more ML APIs (#31404) + Do not preallocate bytes for channel buffer (#31400) + Docs: Advice for reindexing many indices (#31279) + Mute HttpExporterTests#testHttpExporterShutdown test Tracked by #31433 + Docs: Add note about removing prepareExecute from the java client (#31401) + Make release notes ignore the `>test-failure` label. (#31309) + +commit 6ebe6e32cf3d42f0a82e93915e7f9e2b1b9974f7 +Author: David Kyle +Date: Wed Jun 20 16:57:51 2018 +0100 + + [DOCS] Omit shard failures assertion for incompatible responses (#31430) + + Filter out the assertion for _cat and _xpack/ml/datafeed APIs + +commit 8d9b98569f9754f9d87bf5323ba2cfc1b318a677 +Merge: c5b69c87c69 f012de0f00d +Author: Tal Levy +Date: Wed Jun 20 08:30:49 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit f012de0f00d3a6aacf51b122f3c7a0f87010a5f8 +Author: Lisa Cawley +Date: Wed Jun 20 08:17:11 2018 -0700 + + [DOCS] Move licensing APIs to docs (#31445) + +commit 90d62e6e4d8b331a25ab1a99e4addc9b4ef9cbc3 +Author: Daniel Mitterdorfer +Date: Wed Jun 20 16:42:50 2018 +0200 + + Add Delete Snapshot High Level REST API + + With this commit we add the delete snapshot API to the Java high level + REST client. + + Relates #27205 + Relates #31393 + +commit db1b97fd853462b7b5cc44511236de1b9073e494 +Author: Nhat Nguyen +Date: Wed Jun 20 10:34:08 2018 -0400 + + Remove QueryCachingPolicy#ALWAYS_CACHE (#31451) + + The QueryCachingPolicy#ALWAYS_CACHE was deprecated in Lucene-7.4 and + will be removed in Lucene-8.0. This change replaces it with QueryCachingPolicy. + This also makes INDEX_QUERY_CACHE_EVERYTHING_SETTING visible in testing only. + +commit 297e99c4c2bb313e45f320d3ccd68498ddc2c4fd +Author: Jonathan Pool +Date: Wed Jun 20 08:18:55 2018 -0400 + + [Docs] Extend Homebrew installation instructions (#28902) + + Adding a note about proceeding after a successful homebrew installation. + +commit 2aefb72891064e8d4d7c8f440663dc201369c828 +Author: Daniel Mitterdorfer +Date: Wed Jun 20 13:12:56 2018 +0200 + + Choose JVM options ergonomically + + With this commit we add the possibility to define further JVM options (and + system properties) based on the current environment. As a proof of concept, it + chooses Netty's allocator ergonomically based on the maximum defined heap size. + We switch to the unpooled allocator at 1GB heap size (value determined + experimentally, see #30684 for more details). We are also explicit about the + choice of the allocator in either case. + + Relates #30684 + +commit e7a7b9689d8a45bbee71a9ccc5d72b9e91ed56e1 +Author: Peter Dyson +Date: Wed Jun 20 21:04:03 2018 +1000 + + [Docs] Mention ip_range datatypes on ip type page (#31416) + + A link to the ip_range datatype page provides a way for newer users to know + it exists if they land directly on the ip datatype page first via a search. + +commit c5b69c87c6926e04e0a778c413ac4693c19b9d8b +Merge: 8677bfb0c22 5683bc60a69 +Author: Colin Goodheart-Smithe +Date: Wed Jun 20 10:31:45 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 5683bc60a69f928ac37431a8410fc0527381dbbf +Author: Alan Woodward +Date: Wed Jun 20 10:16:26 2018 +0100 + + Multiplexing token filter (#31208) + + The `multiplexer` filter emits multiple tokens at the same position, each + version of the token haivng been passed through a different filter chain. + Identical tokens at the same position are removed. + + This allows users to, for example, index lowercase and original-case tokens, + or stemmed and unstemmed versions, in the same field, so that they can search + for a stemmed term within x positions of an unstemmed term. + +commit df10704ffc7f92dbb64e0660e6604f2b426f65e7 +Author: Christoph Büscher +Date: Wed Jun 20 10:23:39 2018 +0200 + + Fix use of time zone in date_histogram rewrite (#31407) + + Currently, DateHistogramAggregationBuilder#rewriteTimeZone uses the aggregation + date math parser and time zone to check whether all values in a read have the + same timezone to speed up computation. However, the upper and lower bounds to + check are retrieved as longs in epoch_millis, so they don't need to get parsed + using a time zone or a parser other than "epoch_millis". This changes this + behaviour that was causing problems when the field type mapping was specifying + only "epoch_millis" as a format but a different timezone than UTC was used. + + Closes #31392 + +commit 401800d95840a91fb0c8896251d2b6e7972a707c +Author: Ryan Ernst +Date: Tue Jun 19 17:06:09 2018 -0700 + + Core: Remove index name resolver from base TransportAction (#31002) + + Most transport actions don't need to resolve index names. This commit + removes the index name resolver as a super constructor parameter for + TransportAction. The actions that do need the resolver then have a + member added to keep the resolver from their own constructor. + +commit 8677bfb0c2296fa1ab498a08520eec331ddbc4d1 +Author: Tal Levy +Date: Tue Jun 19 15:20:42 2018 -0700 + + fix client signature + +commit 01d456b55cc6ac90ea7716012eeed666440124c0 +Author: Tal Levy +Date: Tue Jun 19 15:01:15 2018 -0700 + + fix actions to reflect changes in master + +commit 5971eb83c434c1e52edbda2c038bf28740bab6d2 +Author: Lisa Cawley +Date: Tue Jun 19 13:57:10 2018 -0700 + + [DOCS] Fixes code snippet testing for machine learning (#31189) + +commit b44e1c197871435f45bbacf2c64768fd3c4c21cf +Author: Sue Gallagher <36747279+Sue-Gallagher@users.noreply.github.com> +Date: Tue Jun 19 13:48:13 2018 -0700 + + [DOCS] Removed and params from MLT. Closes #28128 (#31370) + +commit eee1a451fc3159e2d6f85692a741c85f54cb8a20 +Merge: 5d52f297cc5 dc57eece753 +Author: Tal Levy +Date: Tue Jun 19 11:37:47 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit dc57eece75334b4fb2875082790eda3cf5e5088d +Author: Jay Modi +Date: Tue Jun 19 11:58:34 2018 -0600 + + Security: fix joining cluster with production license (#31341) + + The changes made to disable security for trial licenses unless security + is explicitly enabled caused issues when a 6.3 node attempts to join a + cluster that already has a production license installed. The new node + starts off with a trial license and `xpack.security.enabled` is not + set for the node, which causes the security code to skip attaching the + user to the request. The existing cluster has security enabled and the + lack of a user attached to the requests causes the request to be + rejected. + + This commit changes the security code to check if the state has been + recovered yet when making the decision on whether or not to attach a + user. If the state has not yet been recovered, the code will attach + the user to the request in case security is enabled on the cluster + being joined. + + Closes #31332 + +commit 529e704b1162a232e9714ea49d13e8597f8578ba +Author: Tim Brooks +Date: Tue Jun 19 11:50:03 2018 -0600 + + Unify http channels and exception handling (#31379) + + This is a general cleanup of channels and exception handling in http. + This commit introduces a CloseableChannel that is a superclass of + TcpChannel and HttpChannel. This allows us to unify the closing logic + between tcp and http transports. Additionally, the normal http channels + are extracted to the abstract server transport. + + Finally, this commit (mostly) unifies the exception handling between nio + and netty4 http server transports. + +commit 8fd1f5fbed65ede2e904d29255e3d3b12643b7ae +Author: Lisa Cawley +Date: Tue Jun 19 10:33:57 2018 -0700 + + [DOCS] Moves the info API to docs (#31121) + +commit 40c4bd562844beda9013fde5160143f3bc1f5322 +Author: Yannick Welsch +Date: Tue Jun 19 19:03:31 2018 +0200 + + Preserve response headers on cluster update task (#31421) + + #31241 changed the cluster state update tasks to run under system context. The context wrapping + did not preserve response headers, though. This has led to a test failure on 6.x #31408, as the + deprecation warnings were not carried back anymore to the caller when creating an index. This + commit changes the restorable context supplier to preserve response headers. + +commit cfb470429ea230e5c7b6e440a5480256e2dd7d6c +Author: Lisa Cawley +Date: Tue Jun 19 08:49:32 2018 -0700 + + [DOCS] Add code snippet testing for more ML APIs (#31404) + +commit ffba20b748c80bfa664be9157bc88c7a2b11b360 +Author: Tim Brooks +Date: Tue Jun 19 09:36:12 2018 -0600 + + Do not preallocate bytes for channel buffer (#31400) + + Currently, when we open a new channel, we pass it an + InboundChannelBuffer. The channel buffer is preallocated a single 16kb + page. However, there is no guarantee that this channel will be read from + anytime soon. Instead, this commit does not preallocate that page. That + page will be allocated when we receive a read event. + +commit 5236d0291e7eee0a40f71b1d94c59bc1a31a5cb0 +Author: Nik Everett +Date: Tue Jun 19 11:15:50 2018 -0400 + + Docs: Advice for reindexing many indices (#31279) + + Folks tend to want to be able to make a single `_reindex` call to + migrate many indices. You *can* do that and we even have an example of + how to do that in the docs but it isn't always a good idea. This change + adds some advice to the docs: generally you want to make one reindex + call per index. + + Closes #22920 + +commit 73c182ce089a2f467141b707a3db41e3b972897b +Author: markharwood +Date: Tue Jun 19 15:51:59 2018 +0100 + + Mute HttpExporterTests#testHttpExporterShutdown test + Tracked by #31433 + +commit c74cd30ac6fb354534fbba3cd2ddd76f43e13910 +Author: Nhat Nguyen +Date: Tue Jun 19 10:49:05 2018 -0400 + + Remove request type parameter from CCR actions + + Relates #31405 + +commit c0961b79be3350e00869202f40b88ee39834833a +Author: Ryan Ernst +Date: Tue Jun 19 07:21:58 2018 -0700 + + Docs: Add note about removing prepareExecute from the java client (#31401) + + relates #30966 + +commit ec04366d29eda11963655441f7ee2e0f06c33afe +Merge: 50ce9903057 2396cbd449c +Author: Nhat Nguyen +Date: Tue Jun 19 09:46:39 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add get stored script and delete stored script to high level REST API - post backport fix + Add get stored script and delete stored script to high level REST API (#31355) + Core: Combine Action and GenericAction (#31405) + Fix reference to XContentBuilder.string() (#31337) + Avoid sending duplicate remote failed shard requests (#31313) + Fix defaults in GeoShapeFieldMapper output (#31302) + RestAPI: Reject forcemerge requests with a body (#30792) + Packaging: Remove windows bin files from the tar distribution (#30596) + Docs: Use the default distribution to test docs (#31251) + [DOCS] Adds testing for security APIs (#31345) + Clarify that IP range data can be specified in CIDR notation. (#31374) + Use system context for cluster state update tasks (#31241) + Percentile/Ranks should return null instead of NaN when empty (#30460) + REST high-level client: add validate query API (#31077) + Move language analyzers from server to analysis-common module. (#31300) + [Test] Fix :example-plugins:rest-handler on Windows + Expose lucene's RemoveDuplicatesTokenFilter (#31275) + Reload secure settings for plugins (#31383) + Remove some cases in FieldTypeLookupTests that are no longer relevant. (#31381) + Ensure we don't use a remote profile if cluster name matches (#31331) + [TEST] Double write alias fault (#30942) + [DOCS] Fix version in SQL JDBC Maven template + [DOCS] Improve install and setup section for SQL JDBC + SQL: Fix rest endpoint names in node stats (#31371) + Support for remote path in reindex api - post backport fix Closes #22913 + [ML] Put ML filter API response should contain the filter (#31362) + Support for remote path in reindex api (#31290) + Add byte array pooling to nio http transport (#31349) + Remove trial status info from start trial doc (#31365) + [DOCS] Adds links to release notes and highlights + add is-write-index flag to aliases (#30942) + Add rollover-creation-date setting to rolled over index (#31144) + [ML] Hold ML filter items in sorted set (#31338) + [Tests] Fix edge case in ScriptedMetricAggregatorTests (#31357) + +commit a5540ba19c8364289b699de5262e34c1a30f01af +Author: Adrien Grand +Date: Tue Jun 19 15:43:31 2018 +0200 + + Make release notes ignore the `>test-failure` label. (#31309) + + This label is uninteresting for release notes. + +commit 2396cbd449ce1e811ca43f29042151c5f8f5d79a +Author: Vladimir Dolzhenko +Date: Tue Jun 19 14:21:11 2018 +0200 + + Add get stored script and delete stored script to high level REST API - post backport fix + + Relates to #27205 + +commit 04e4e44409d8fe2d98793f8f137892e39a113c1f +Author: Vladimir Dolzhenko +Date: Tue Jun 19 14:21:11 2018 +0200 + + Add get stored script and delete stored script to high level REST API (#31355) + + Add get stored script and delete stored script to high level REST API + + Relates to #27205 + +commit 50ce990305704b4ab055f9cd586f665dbbfc37d3 +Author: Martijn van Groningen +Date: Tue Jun 19 10:22:42 2018 +0200 + + added missing serialization tests + +commit e67aa96c81f309ebedc1d529e255b48069b262d1 +Author: Ryan Ernst +Date: Mon Jun 18 23:53:04 2018 +0200 + + Core: Combine Action and GenericAction (#31405) + + Since #30966, Action no longer has anything but a call to the + GenericAction super constructor. This commit renames GenericAction + into Action, thus eliminating the Action class. Additionally, this + commit removes the Request generic parameter of the class, since + it was unused. + +commit 4ad334f8e0af43f6aa38bd8dbe7bc7227cd9eb11 +Author: Lee Hinman +Date: Mon Jun 18 15:04:26 2018 -0600 + + Fix reference to XContentBuilder.string() (#31337) + + In 6.3 this was moved to `Strings.toString(XContentBuilder)` as part of the + XContent extraction. This commit fixes the docs to reference the new method. + + Resolves #31326 + +commit 2a8381d3fa5c8d88aff662d3c1066a6e258578c2 +Author: Nhat Nguyen +Date: Mon Jun 18 15:05:34 2018 -0400 + + Avoid sending duplicate remote failed shard requests (#31313) + + Today if a write replication request fails, we will send a shard-failed + message to the master node to fail that replica. However, if there are + many ongoing write replication requests and the master node is busy, we + might overwhelm the cluster and the master node with many shard-failed + requests. + + This commit tries to minimize the shard-failed requests in the above + scenario by caching the ongoing shard-failed requests. + + This issue was discussed at + https://discuss.elastic.co/t/half-dead-node-lead-to-cluster-hang/113658/25. + +commit d9a6d69a0ddcb434811705067b40a0630edea3de +Author: Igor Motov +Date: Mon Jun 18 13:50:52 2018 -0400 + + Fix defaults in GeoShapeFieldMapper output (#31302) + + GeoShapeFieldMapper should show actual defaults instead of placeholder + values when the mapping is requested with include_defaults=true. + + Closes #23206 + +commit 5d52f297cc570a2432b038832133f9ffb56d3bab +Merge: aa6944a7196 340313b0482 +Author: Tal Levy +Date: Mon Jun 18 10:06:47 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 340313b0482dbddde07b64172d668d27277e35bb +Author: Ryan Ernst +Date: Mon Jun 18 19:03:46 2018 +0200 + + RestAPI: Reject forcemerge requests with a body (#30792) + + This commit adds validation to forcemerge rest requests which contain a + body. All parameters to force merge must be part of http params. + + closes #29584 + +commit f3297ed23a9472e2f184b0ac4b1f367daef44cbc +Author: Ryan Ernst +Date: Mon Jun 18 19:02:51 2018 +0200 + + Packaging: Remove windows bin files from the tar distribution (#30596) + + This commit removes windows specific files from the tar distribution. + Windows users use the zip, linux users use the tar. + +commit 73549281e8443a0b579b3f606a41b720465d8054 +Author: Nik Everett +Date: Mon Jun 18 12:06:42 2018 -0400 + + Docs: Use the default distribution to test docs (#31251) + + This switches the docs tests from the `oss-zip` distribution to the + `zip` distribution so they have xpack installed and configured with the + default basic license. The goal is to be able to merge the + `x-pack/docs` directory into the `docs` directory, marking the x-pack + docs with some kind of marker. This is the first step in that process. + + This also enables `-Dtests.distribution` support for the `docs` + directory so you can run the tests against the `oss-zip` distribution + with something like + ``` + ./gradlew -p docs check -Dtests.distribution=oss-zip + ``` + + We can set up Jenkins to run both. + + Relates to #30665 + +commit ea92864eb179cb68af3f2a774713527f493f550b +Author: Lisa Cawley +Date: Mon Jun 18 08:48:23 2018 -0700 + + [DOCS] Adds testing for security APIs (#31345) + +commit 3f5ebb862de56efed4edcdb8b338590cb54be6f5 +Author: Julie Tibshirani +Date: Mon Jun 18 08:21:41 2018 -0700 + + Clarify that IP range data can be specified in CIDR notation. (#31374) + +commit 02a4ef38a7960895b4e26111167676ec49302cba +Author: Yannick Welsch +Date: Mon Jun 18 16:46:04 2018 +0200 + + Use system context for cluster state update tasks (#31241) + + This commit makes it so that cluster state update tasks always run under the system context, only + restoring the original context when the listener that was provided with the task is called. A notable + exception is the clusterStatePublished(...) callback which will still run under system context, + because it's defined on the executor-level, and not the task level, and only called once for the + combined batch of tasks and can therefore not be uniquely identified with a task / thread context. + + Relates #30603 + +commit 1502812c1a4b7dd08417a45798b8fe66311e7028 +Author: Zachary Tong +Date: Mon Jun 18 10:01:28 2018 -0400 + + Percentile/Ranks should return null instead of NaN when empty (#30460) + + The other metric aggregations (min/max/etc) return `null` as their XContent value and string when nothing was computed (due to empty/missing fields). Percentiles and Percentile Ranks, however, return `NaN `which is inconsistent and confusing for the user. This fixes the inconsistency by making the aggs return `null`. This applies to both the numeric value and the "as string" value. + + Note: like the metric aggs, this does not change the value if fetched directly from the percentiles object, which will return as `NaN`/`"NaN"`. This only changes the XContent output. + + While this is a bugfix, it still breaks bwc in a minor way as the response changes from prior version. + + Closes #29066 + +commit c4f8df3ad6b8657557a7efb9399e70927e1f0340 +Author: Sohaib Iftikhar +Date: Mon Jun 18 15:59:29 2018 +0200 + + REST high-level client: add validate query API (#31077) + + Adds the validate query API to the high level rest client. + +commit 47095357bc1ae0df9a19ea328a64745b11aa2e08 +Author: Martijn van Groningen +Date: Mon Jun 18 11:24:43 2018 +0200 + + Move language analyzers from server to analysis-common module. (#31300) + + The following analyzers were moved from server module to analysis-common module: + `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `latvian`, + `lithuanian`, `norwegian`, `persian`, `portuguese`, `romanian`, `russian`, + `sorani`, `spanish`, `swedish`, `turkish` and `thai`. + + Relates to #23658 + +commit f5135050526d5ba3febcb0b43a7dd708d49a4c0f +Author: Tanguy Leroux +Date: Mon Jun 18 10:49:22 2018 +0200 + + [Test] Fix :example-plugins:rest-handler on Windows + +commit 8c0ec05a1225c76a4ae9e6592ac38e63e5ed4dc8 +Author: Alan Woodward +Date: Mon Jun 18 09:46:12 2018 +0100 + + Expose lucene's RemoveDuplicatesTokenFilter (#31275) + +commit 3378240b2954fcf879ac84f440002685c4b30cc9 +Author: Albert Zaharovits +Date: Mon Jun 18 09:42:11 2018 +0300 + + Reload secure settings for plugins (#31383) + + Adds the ability to reread and decrypt the local node keystore. + Commonly, the contents of the keystore, backing the `SecureSettings`, + are not retrievable except during node initialization. This changes that + by adding a new API which broadcasts a password to every node. The + password is used to decrypt the local keystore and use it to populate + a `Settings` object that is passes to all the plugins implementing the + `ReloadablePlugin` interface. The plugin is then responsible to do + whatever "reload" means in his case. When the `reload`handler returns, + the keystore is closed and its contents are no longer retrievable. + Password is never stored persistently on any node. + Plugins that have been moded in this commit are: `repository-azure`, + `repository-s3`, `repository-gcs` and `discovery-ec2`. + +commit 16fa6b270f5ebe9e78e0d283a88f4f7ced71c35a +Author: Julie Tibshirani +Date: Sun Jun 17 21:42:42 2018 -0700 + + Remove some cases in FieldTypeLookupTests that are no longer relevant. (#31381) + +commit 3d5f113ada63b9175631e6ad1ef1645dc8f5f275 +Author: Simon Willnauer +Date: Sun Jun 17 13:32:53 2018 +0200 + + Ensure we don't use a remote profile if cluster name matches (#31331) + + If we are running into a race condition between a node being configured + to be a remote node for cross cluster search etc. and that node joining + the cluster we might connect to that node with a remote profile. If that + node now joins the cluster it connected to it as a CCS remote node we use + the wrong profile and can't use bulk connections etc. anymore. This change + uses the remote profile only if we connect to a node that has a different cluster + name than the local cluster. This is not a perfect fix for this situation but + is the safe option while potentially only loose a small optimization of using + less connections per node which is small anyways since we only connect to a + small set of nodes. + + Closes #29321 + +commit 5b94afd3099cd60c7342ad7ba4e5764420d531eb +Author: Albert Zaharovits +Date: Sun Jun 17 12:17:28 2018 +0300 + + [TEST] Double write alias fault (#30942) + +commit bbcfcd1ca5a6b78757a7f5578da0cc618839c27f +Author: Costin Leau +Date: Sun Jun 17 00:40:01 2018 +0300 + + [DOCS] Fix version in SQL JDBC Maven template + +commit 215c5f292a9b954fa7ccfe3f29540875076ae1ae +Author: Costin Leau +Date: Sun Jun 17 00:14:59 2018 +0300 + + [DOCS] Improve install and setup section for SQL JDBC + +commit c3084a332b6d2f34fa069d969932d53aacf1e5f7 +Author: Igor Motov +Date: Fri Jun 15 19:07:47 2018 -0400 + + SQL: Fix rest endpoint names in node stats (#31371) + + Fixes wrong name for the sql translate endpoint and makes rest endpoint + names in stats more consistent. + +commit babb16d90cdc05b1b46d91948f7d6bf38a49bf6e +Author: Vladimir Dolzhenko +Date: Fri Jun 15 22:24:47 2018 +0200 + + Support for remote path in reindex api - post backport fix + Closes #22913 + +commit c6a5a6d924676aa5b4678ef617fc2c4e966f2b60 +Author: Dimitris Athanasiou +Date: Fri Jun 15 21:15:35 2018 +0100 + + [ML] Put ML filter API response should contain the filter (#31362) + +commit dbc9d602608864f16bf59741114f4c05119b73c6 +Author: Vladimir Dolzhenko +Date: Fri Jun 15 22:14:28 2018 +0200 + + Support for remote path in reindex api (#31290) + + Support for remote path in reindex api + Closes #22913 + +commit a705e1a9e30abe2b636ab38602bd1afea2b0c422 +Author: Tim Brooks +Date: Fri Jun 15 14:01:03 2018 -0600 + + Add byte array pooling to nio http transport (#31349) + + This is related to #28898. This PR implements pooling of bytes arrays + when reading from the wire in the http server transport. In order to do + this, we must integrate with netty reference counting. That manner in + which this PR implements this is making Pages in InboundChannelBuffer + reference counted. When we accessing the underlying page to pass to + netty, we retain the page. When netty releases its bytebuf, it releases + the underlying pages we have passed to it. + +commit 605dbbeabdd6769daa79a3afa4955e0c7b2671cb +Author: Tim Brooks +Date: Fri Jun 15 12:39:25 2018 -0600 + + Remove trial status info from start trial doc (#31365) + + This is related to #31325. There is currently information about the + get-trial-status api on the start-trial api documentation page. It also + has the incorrect route for that api. This commit removes that + information as the start-trial page properly links to a page providing + documenation about get-trial-status. + +commit aa6944a7196b8423aa9b7c2cee8e2d906183a005 +Author: Tal Levy +Date: Fri Jun 15 11:39:19 2018 -0700 + + render non-ElasticsearchException in ILM (#31284) + + ILM was rendering exceptions using the exception helper that + basically ignores simply rendering non-elasticsearch exceptions when + no details are desired. This commit updates the method used to still be + a rather simple rendering of the exception. The rendering lacks + all the causes, but does a sufficient job in rendering the top-level + message for one of our most expected exceptions... IllegalArgumentException + +commit 01939794fe0e2a97707aaa45dead91219f8a5a3f +Author: Tal Levy +Date: Fri Jun 15 11:38:53 2018 -0700 + + Introduce Maintenance Mode to ILM (#31164) + + This PR introduces a concept of a maintenance mode for the + lifecycle service. During maintenance mode, no policies are + executed. + + To be placed into maintenance mode, users must first issue a + request to be placed in maintenance mode. Once the service + is assured that no policies are in actions that are not to be + interrupted (like ShrinkAction), the service will place itself + in maintenance mode. + + APIs to-be introduced: + + - POST _xpack/index_lifecycle/maintenance/_request + - issues a request to be placed into maintenenance mode. + This is not immediate, since we must first verify that + it is safe to go from REQUESTED -> IN maintenance mode. + - POST _xpack/index_lifecycle/maintenance/_stop + - issues a request to be taken out (this is immediate) + - GET _xpack/index_lifecycle/maintenance + - get back the current mode our lifecycle management is in + +commit 4918ae6fe480f06f28949b365cb6de0ebbc2bb88 +Author: lcawl +Date: Fri Jun 15 11:04:11 2018 -0700 + + [DOCS] Adds links to release notes and highlights + +commit c54937731e9abbc1a0ae8145416fab34a3393198 +Merge: fe4d74cc943 3b70e943eb0 +Author: Tal Levy +Date: Fri Jun 15 10:16:05 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit fe4d74cc9439f6f64b3f377d6bb588c51ec3983c +Author: Tal Levy +Date: Fri Jun 15 09:29:07 2018 -0700 + + fix merge messup + +commit 3b70e943eb08f90aa24e9c34033c8ca2a7211239 +Author: Tal Levy +Date: Fri Jun 15 08:45:29 2018 -0700 + + add is-write-index flag to aliases (#30942) + + This commit adds the is-write-index flag for aliases. + It allows requests to set the flag, and responses to display the flag. + It does not validate and/or affect any indexing/getting/updating behavior + of Elasticsearch -- this will be done in a follow-up PR. + +commit eda4964f640f9618ec8c80db64904ef35426fd28 +Author: Tal Levy +Date: Fri Jun 15 08:44:29 2018 -0700 + + Add rollover-creation-date setting to rolled over index (#31144) + + This commit introduces a new property to IndexMetaData called + RolloverInfo. This object contains a map containing the aliases + that were used to rollover the related index, which conditions + were met, and at what time the rollover took place. + + much like the `index.creation_date`, it captures the approximate time + that the index was rolled over to a new one. + +commit da5bfda5f314b95c3cce289c0ddb72208307f43a +Author: Dimitris Athanasiou +Date: Fri Jun 15 16:29:09 2018 +0100 + + [ML] Hold ML filter items in sorted set (#31338) + + Filter items should be unique. They should also + be sorted to make them easier to read plus save + sorting in the autodetect process. + +commit 85d0613ea6d72671e72c8cb993e1526b20cf3fbb +Merge: 80ab7739ca8 fec7860edcc +Author: Tal Levy +Date: Fri Jun 15 08:28:23 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 80ab7739ca8877e2036edfac724f6014fefb7e52 +Author: Tal Levy +Date: Fri Jun 15 08:23:39 2018 -0700 + + Move IndexLifecycleMetadata installation to put-lifecycle-action (#31346) + + There is a problematic scenario with x-pack-cluster master-nodes + attempting to install custom metadata into the cluster-state and + broadcasting that to non-x-pack-enabled nodes. Since those nodes + are not aware of this custom metadata, their cluster-state recovery + will be broken. This change ensures that newly-elected x-pack master + nodes bootstrap IndexLifecycleMetadata upon the first request to + leverage its features. This means that PutLifecycleAction is + now responsible for installing the metadata. Since this X-Pack API + can only be called once all nodes in the cluster have x-pack enabled, + it is safe to assume that the cluster will appropriately handle the + cluster-state recovery with the new set of index-lifecycle metadata. + +commit a8abf0fcc004005648e779806ae978dea81d804a +Merge: 73c9dd976b6 8453ca638d4 +Author: Nhat Nguyen +Date: Fri Jun 15 11:19:35 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Upgrade to Lucene-7.4.0-snapshot-518d303506 (#31360) + Rankeval: Fold template test project into main module (#31203) + Add QA project and fixture based test for discovery-ec2 plugin (#31107) + [Docs] Remove reference to repository-s3 plugin creating an S3 bucket (#31359) + REST Client: NodeSelector for node attributes (#31296) + LLClient: Fix assertion on windows + Add details section for dcg ranking metric (#31177) + [ML] Re-enable tests muted in #30982 + +commit fec7860edccd60a4e11d249367c7289281154920 +Author: Christoph Büscher +Date: Fri Jun 15 17:12:42 2018 +0200 + + [Tests] Fix edge case in ScriptedMetricAggregatorTests (#31357) + + An expected exception is only thrown when there are documents in the index + created in the test setup. Fixed the test by making sure there is at least one. + + Closes #31307 + +commit 8453ca638d4d8a0d82628484a2a8c0183929b066 +Author: Nhat Nguyen +Date: Fri Jun 15 10:58:21 2018 -0400 + + Upgrade to Lucene-7.4.0-snapshot-518d303506 (#31360) + +commit 02346c20a2022a927d88ed3b6819634d7b35140e +Author: Christoph Büscher +Date: Fri Jun 15 15:55:39 2018 +0200 + + Rankeval: Fold template test project into main module (#31203) + + This change moves tests in `smoke-test-rank-eval-with-mustache` into the main + ranking evaluation module by declaring that the integration testing cluster + requires the `lang-mustache` plugin. This avoids having to maintain the qa + project for only one basic test suite. + +commit afc91e92fb697a3241b26a5bb7cdc0a3769f4085 +Author: Tanguy Leroux +Date: Fri Jun 15 15:49:14 2018 +0200 + + Add QA project and fixture based test for discovery-ec2 plugin (#31107) + + This commit adds a new QA sub project to the discovery-ec2 plugin. + This project uses a fixture to test the plugin using a multi-node cluster. + Once all nodes are started, the nodes transport addresses are written + in a file that is later read by the fixture. + +commit 3274e7fd1a6ca22e3a642f415554674e8030fe85 +Author: Tanguy Leroux +Date: Fri Jun 15 14:55:14 2018 +0200 + + [Docs] Remove reference to repository-s3 plugin creating an S3 bucket (#31359) + + Closes #30910 + +commit 856936c2863799f250818b4b7582aa707a4dcf92 +Author: Nik Everett +Date: Fri Jun 15 08:04:54 2018 -0400 + + REST Client: NodeSelector for node attributes (#31296) + + Add a `NodeSelector` so that users can filter the nodes that receive + requests based on node attributes. + + I believe we'll need this to backport #30523 and we want it anyway. + + I also added a bash script to help with rebuilding the sniffer parsing + test documents. + +commit 045f76d67f37f315fc42a3960ce88500e105f0d6 +Author: Nik Everett +Date: Fri Jun 15 08:02:47 2018 -0400 + + LLClient: Fix assertion on windows + + In windows the exception message is ever so slightly differant than in + Linux and OSX. That is fine. We'll just catch either. + +commit 73c9dd976b65d4e3787fe234ddcd8258cb8b7fa3 +Author: Martijn van Groningen +Date: Fri Jun 15 12:32:08 2018 +0200 + + Remove action request builders. + +commit a0d6c19e7568077ad97a047c9b985c468ed6a465 +Author: Christoph Büscher +Date: Fri Jun 15 11:56:16 2018 +0200 + + Add details section for dcg ranking metric (#31177) + + While the other two ranking evaluation metrics (precicion and reciprocal rank) + already provide a more detailed output for how their score is calculated, the + discounted cumulative gain metric (dcg) and its normalized variant are lacking + this until now. Its not really clear which level of detail might be useful for + debugging and understanding the final metric calculation, but this change adds a + `metric_details` section to REST output that contains some information about the + evaluation details. + +commit ca00deb8addf6cfc5f18ec0e7abd18236f9dc6c3 +Author: David Kyle +Date: Fri Jun 15 10:54:13 2018 +0100 + + [ML] Re-enable tests muted in #30982 + +commit 18938aab39b6cc212b73698aa83eecfdda40159b +Author: Tanguy Leroux +Date: Fri Jun 15 11:46:08 2018 +0200 + + Adapt ShardFollowTasksExecutor after #31031 + +commit 9c03b4844f5cffdec053187ce4b1e1792436365b +Merge: cc824ebb5e5 992c7889ee8 +Author: Tanguy Leroux +Date: Fri Jun 15 10:40:31 2018 +0200 + + Merge branch 'master' into ccr + + * master: + 992c7889ee8 Uncouple persistent task state and status (#31031) + 8c6ee7db54f Describe how to add a plugin in Dockerfile (#31340) + 1c5cec0ac7e Remove http status code maps (#31350) + 87a676e4d53 Do not set vm.max_map_count when unnecessary (#31285) + e5b71375082 TEST: getCapturedRequestsAndClear should be atomic (#31312) + 03241037373 Painless: Fix bug for static method calls on interfaces (#31348) + d6d0727aac0 QA: Fix resolution of default distribution (#31351) + fcf1e41e429 Extract common http logic to server (#31311) + 6dd81ead74f Build: Fix the license in the pom zip and tar (#31336) + 8f886cd4be8 Treat ack timeout more like a publish timeout (#31303) + 9b293275af8 [ML] Add description to ML filters (#31330) + f7a0cafe557 SQL: Fix build on Java 10 + 375d09c588a [TEST] Fix RemoteClusterClientTests#testEnsureWeReconnect + 4877cec3e82 More detailed tracing when writing metadata (#31319) + bbfe1eccc79 [Tests] Mutualize fixtures code in BaseHttpFixture (#31210) + +commit 992c7889ee83fc3514145d0875ac359010f623ae +Author: Tanguy Leroux +Date: Fri Jun 15 09:26:47 2018 +0200 + + Uncouple persistent task state and status (#31031) + + This pull request removes the relationship between the state + of persistent task (as stored in the cluster state) and the status + of the task (as reported by the Task APIs and used in various + places) that have been confusing for some time (#29608). + + In order to do that, a new PersistentTaskState interface is added. + This interface represents the persisted state of a persistent task. + The methods used to update the state of persistent tasks are + renamed: updatePersistentStatus() becomes updatePersistentTaskState() + and now takes a PersistentTaskState as a parameter. The + Task.Status type as been changed to PersistentTaskState in all + places were it make sense (in persistent task customs in cluster + state and all other methods that deal with the state of an allocated + persistent task). + +commit cc824ebb5e5041b19dcae8bc731ffca336f18ce8 +Author: Martijn van Groningen +Date: Fri Jun 15 07:39:53 2018 +0200 + + [CCR] Added more validation to follow index api. (#31068) + +commit 8c6ee7db54fa3156e9eed968d23a9bce55694e0e +Author: David Pilato +Date: Fri Jun 15 07:32:49 2018 +0200 + + Describe how to add a plugin in Dockerfile (#31340) + + When installing a plugin, people need to add the `--batch` option. + It's better to document it as it could be a common use case. + +commit 1c5cec0ac7ea6cba5faf25de99de89af46e0954e +Author: Tim Brooks +Date: Thu Jun 14 20:16:40 2018 -0600 + + Remove http status code maps (#31350) + + Currently we maintain a compatibility map of http status codes in both + the netty4 and nio modules. These maps convert a RestStatus to a netty + HttpResponseStatus. However, as these fundamentally represent integers, + we can just use the netty valueOf method to convert a RestStatus to a + HttpResponseStatus. + +commit 87a676e4d538a1533964bfe034f88782ca20cbfb +Author: Ben Abrams +Date: Thu Jun 14 18:41:02 2018 -0700 + + Do not set vm.max_map_count when unnecessary (#31285) + + This commit modifies the Sys V init startup scripts to only modify + vm.max_map_count if needed. In this case, needed means that the current + value is less than our default value of 262144 maps. + +commit e5b7137508247b6490e42bbde6fa76f51c5ef343 +Author: Nhat Nguyen +Date: Thu Jun 14 21:32:07 2018 -0400 + + TEST: getCapturedRequestsAndClear should be atomic (#31312) + + We might lose messages between getCapturedRequestsAndClear calls. + This commit makes sure that both getCapturedRequestsAndClear and + getCapturedRequestsByTargetNodeAndClear are atomic. + +commit 03241037373a727444cd7fda2ab23825a6eb84d6 +Author: Jack Conradson +Date: Thu Jun 14 18:30:37 2018 -0700 + + Painless: Fix bug for static method calls on interfaces (#31348) + + Static method calls on interfaces were not being called correctly + which was causing JVM crashes. This change fixes the issue. + +commit 2af05e5480ab63e5602ff92ded3ca5d4d127b08b +Merge: 7412943f604 d6d0727aac0 +Author: Tal Levy +Date: Thu Jun 14 15:32:41 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit d6d0727aac0159a926b80ecce95913b928834267 +Author: Nik Everett +Date: Thu Jun 14 18:25:47 2018 -0400 + + QA: Fix resolution of default distribution (#31351) + + If you run `./gradlew -p qa bwcTest -Dtests.distribution=zip` then we + need to resolve older versions of the default distribution. Since those + aren't available in maven central, we need add the elastic maven repo to + the project. + +commit fcf1e41e429b10e03c5cf9b8551636df7519b4c5 +Author: Tim Brooks +Date: Thu Jun 14 15:10:02 2018 -0600 + + Extract common http logic to server (#31311) + + This is related to #28898. With the addition of the http nio transport, + we now have two different modules that provide http transports. + Currently most of the http logic lives at the module level. However, + some of this logic can live in server. In particular, some of the + setting of headers, cors, and pipelining. This commit begins this moving + in that direction by introducing lower level abstraction (HttpChannel, + HttpRequest, and HttpResonse) that is implemented by the modules. The + higher level rest request and rest channel work can live entirely in + server. + +commit 6dd81ead74fd54247198d380b98053124a30f1ee +Author: Nik Everett +Date: Thu Jun 14 16:22:00 2018 -0400 + + Build: Fix the license in the pom zip and tar (#31336) + + For 6.3 we renamed the `tar` and `zip` distributions to `oss-tar` and + `oss-zip`. Then we added new `tar` and `zip` distributions that contain + x-pack and are licensed under the Elastic License. Unfortunately we + accidentally generated POM files along side the new `tar` and `zip` + distributions that incorrectly claimed that they were Apache 2 licensed. + Oooops. + + This fixes the license on the POMs generated for the `tar` and `zip` + distributions. + +commit 8f886cd4be82ff30e98d812a1deac0a70ad92bc3 +Author: Yannick Welsch +Date: Thu Jun 14 18:32:35 2018 +0200 + + Treat ack timeout more like a publish timeout (#31303) + + This commit changes the ack timeout mechanism so that its behavior is closer to the publish + timeout, i.e., it only comes into play after committing a cluster state. This ensures for example that + an index creation request with a low (ack) timeout value does not return before the cluster state + that contains information about the newly created index is even committed. + +commit 9b293275af8990917ee2152dbf53d8642bda264d +Author: Dimitris Athanasiou +Date: Thu Jun 14 16:52:32 2018 +0100 + + [ML] Add description to ML filters (#31330) + + This adds a `description` to ML filters in order + to allow users to describe their filters in a human + readable form which is also editable (filter updates + to be added shortly). + +commit 1ccb34ac7752c0dc9c50de8a3b7d21c56060f8e6 +Author: Nhat Nguyen +Date: Thu Jun 14 11:44:20 2018 -0400 + + Remove unused imports + +commit f7a0cafe557275cf1d3b6f87209f9c41ecb39b8d +Author: Costin Leau +Date: Thu Jun 14 18:07:29 2018 +0300 + + SQL: Fix build on Java 10 + + Due to a runtime classpath clash, featureAware task was failing on JVMs + higher than 1.8 (since the ASM version from Painless was used instead + which does not recognized Java 9 or 10 bytecode) causing the task to + fail. + This commit excludes the ASM dependency (since it's not used by SQL + itself). + +commit 2cffd85f7fdb791969aac6a8ae821830fd6adc7e +Merge: b4469f82b9d ce245a73200 +Author: Nhat Nguyen +Date: Thu Jun 14 10:45:05 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Remove RestGetAllAliasesAction (#31308) + Temporary fix for broken build + Reenable Checkstyle's unused import rule (#31270) + Remove remaining unused imports before merging #31270 + Fix non-REST doc snippet + [DOC] Extend SQL docs + Immediately flush channel after writing to buffer (#31301) + [DOCS] Shortens ML API intros + Use quotes in the call invocation (#31249) + move security ingest processors to a sub ingest directory (#31306) + Add 5.6.11 version constant. + Fix version detection. + SQL: Whitelist SQL utility class for better scripting (#30681) + [Docs] All Rollup docs experimental, agg limitations, clarify DeleteJob (#31299) + CCS: don't proxy requests for already connected node (#31273) + Mute ScriptedMetricAggregatorTests testSelfReferencingAggStateAfterMap + [test] opensuse packaging turn up debug logging + Add unreleased version 6.3.1 + Removes experimental tag from scripted_metric aggregation (#31298) + [Rollup] Metric config parser must use builder so validation runs (#31159) + [ML] Check licence when datafeeds use cross cluster search (#31247) + Add notion of internal index settings (#31286) + Test: Remove broken yml test feature (#31255) + REST hl client: cluster health to default to cluster level (#31268) + [ML] Update test thresholds to account for changes to memory control (#31289) + Log warnings when cluster state publication failed to some nodes (#31233) + Fix AntFixture waiting condition (#31272) + Ignore numeric shard count if waiting for ALL (#31265) + [ML] Implement new rules design (#31110) + index_prefixes back-compat should test 6.3 (#30951) + Core: Remove plain execute method on TransportAction (#30998) + Update checkstyle to 8.10.1 (#31269) + Set analyzer version in PreBuiltAnalyzerProviderFactory (#31202) + Modify pipelining handlers to require full requests (#31280) + Revert upgrade to Netty 4.1.25.Final (#31282) + Use armored input stream for reading public key (#31229) + Fix Netty 4 Server Transport tests. Again. + REST hl client: adjust wait_for_active_shards param in cluster health (#31266) + REST high-level Client: remove deprecated API methods (#31200) + [DOCS] Mark SQL feature as experimental + [DOCS] Updates machine learning custom URL screenshots (#31222) + Fix naming conventions check for XPackTestCase + Fix security Netty 4 transport tests + Fix race in clear scroll (#31259) + [DOCS] Clarify audit index settings when remote indexing (#30923) + Delete typos in SAML docs (#31199) + REST high-level client: add Cluster Health API (#29331) + [ML][TEST] Mute tests using rules (#31204) + Support RequestedAuthnContext (#31238) + SyncedFlushResponse to implement ToXContentObject (#31155) + Add Get Aliases API to the high-level REST client (#28799) + Remove some line length supressions (#31209) + Validate xContentType in PutWatchRequest. (#31088) + [INGEST] Interrupt the current thread if evaluation grok expressions take too long (#31024) + Suppress extras FS on caching directory tests + Revert "[DOCS] Added 6.3 info & updated the upgrade table. (#30940)" + Revert "Fix snippets in upgrade docs" + Fix snippets in upgrade docs + [DOCS] Added 6.3 info & updated the upgrade table. (#30940) + LLClient: Support host selection (#30523) + Upgrade to Netty 4.1.25.Final (#31232) + Enable custom credentials for core REST tests (#31235) + Move ESIndexLevelReplicationTestCase to test framework (#31243) + Encapsulate Translog in Engine (#31220) + HLRest: Add get index templates API (#31161) + Remove all unused imports and fix CRLF (#31207) + [Tests] Fix self-referencing tests + [TEST] Fix testRecoveryAfterPrimaryPromotion + [Docs] Remove mention pattern files in Grok processor (#31170) + Use stronger write-once semantics for Azure repository (#30437) + Don't swallow exceptions on replication (#31179) + Limit the number of concurrent requests per node (#31206) + Call ensureNoSelfReferences() on _agg state variable after scripted metric agg script executions (#31044) + Move java version checker back to its own jar (#30708) + [test] add fix for rare virtualbox error (#31212) + +commit 375d09c588a58a0f490bac29fb535922819e3453 +Author: Simon Willnauer +Date: Thu Jun 14 16:21:28 2018 +0200 + + [TEST] Fix RemoteClusterClientTests#testEnsureWeReconnect + + Closes #29547 + +commit 7412943f6043fb2fec3d291669be5252b08ff796 +Author: Colin Goodheart-Smithe +Date: Thu Jun 14 13:47:16 2018 +0100 + + Removes unused stepsRegistry from move to step task + +commit 2f00ef777ffaabc02f9dc03c3d52a029ba637d11 +Merge: d794253fc5c bbfe1eccc79 +Author: Colin Goodheart-Smithe +Date: Thu Jun 14 13:43:39 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit d794253fc5cf78d3c56eda6aa51ca60381a8be6c +Author: Colin Goodheart-Smithe +Date: Thu Jun 14 13:42:12 2018 +0100 + + Removes warnings from ILM code + +commit 4877cec3e82554cfc7a03f3552b3c62198af0d28 +Author: David Turner +Date: Thu Jun 14 13:41:25 2018 +0100 + + More detailed tracing when writing metadata (#31319) + + Packaging tests are occasionally failing (#30295) because of very slow index + template creation. It looks like the slow part is updating the on-disk cluster + state, and this change will help to confirm this. + +commit bbfe1eccc79f700d5806c959c8730aa804e1068d +Author: Tanguy Leroux +Date: Thu Jun 14 14:09:56 2018 +0200 + + [Tests] Mutualize fixtures code in BaseHttpFixture (#31210) + + Many fixtures have similar code for writing the pid & ports files or + for handling HTTP requests. This commit adds an AbstractHttpFixture + class in the test framework that can be extended for specific testing purposes. + +commit ce38127a16c95644557c790276edffb5492def0c +Merge: 8ac260d79f2 ce245a73200 +Author: Colin Goodheart-Smithe +Date: Thu Jun 14 13:02:24 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 8ac260d79f25829d988e1028f78d2bba0a3585d5 +Author: Colin Goodheart-Smithe +Date: Thu Jun 14 11:33:56 2018 +0100 + + Removes erroreously added file + +commit ce245a7320062170668e5056994defad45375c4f +Author: Luca Cavanna +Date: Thu Jun 14 11:21:16 2018 +0200 + + Remove RestGetAllAliasesAction (#31308) + + We currently have a specific REST action to retrieve all aliaes, which + uses internally the get index API. This doesn't seem to be required + anymore though as the existing RestGetAliaesAction could as well take + the requests with no indices and aliases specified. + + This commit removes the RestGetAllAliasesAction in favour of using + RestGetAliasesAction also for requests that don't specify indices nor + aliases. Similar to #31129. + +commit 89a24698b1e60abc0d5021a02bbc4bc411fdc59c +Author: Tim Vernum +Date: Thu Jun 14 18:22:24 2018 +1000 + + Temporary fix for broken build + + x-pack/sql depends on lang-painless which depends on ASM 5.1 + FeatureAwareCheck needs ASM 6 + This is a hack to strip ASM5 from the classpath for FeatureAwareCheck + +commit 4d7447cb5e67f2739d5edc374e9c7c3a46d414da +Author: Tanguy Leroux +Date: Thu Jun 14 09:52:46 2018 +0200 + + Reenable Checkstyle's unused import rule (#31270) + +commit 2d4c9ce08c906a4f2b19179cd1f8244ac226c03e +Author: Tanguy Leroux +Date: Thu Jun 14 09:52:03 2018 +0200 + + Remove remaining unused imports before merging #31270 + +commit 74c6f18feab073a1f8c3a09630fff0b5bae3e28f +Author: Tim Vernum +Date: Thu Jun 14 16:26:17 2018 +1000 + + Fix non-REST doc snippet + +commit b9e1a905eb1fbd6c1946889afbc236c9cc065e22 +Author: Tal Levy +Date: Wed Jun 13 16:04:54 2018 -0700 + + fix retryrequest test + +commit 11ef3b90a32fd853fbc45b316ae33d6f334bc336 +Merge: 04e0d0a5e77 870a913217b +Author: Tal Levy +Date: Wed Jun 13 15:56:47 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 870a913217be6de41c4f0a91c9fa5493017c8554 +Author: Costin Leau +Date: Wed Jun 13 13:53:08 2018 +0300 + + [DOC] Extend SQL docs + + Add overview section + Add data type section + Improve function section + +commit 700357d04e554f3b4fc5098e442b28eeeb49f9eb +Author: Tim Brooks +Date: Wed Jun 13 15:32:13 2018 -0600 + + Immediately flush channel after writing to buffer (#31301) + + This is related to #27260. Currently when we queue a write with a + channel we set OP_WRITE and wait until the next selection loop to flush + the write. However, if the channel does not have a pending write, it + is probably ready to flush. This PR implements an optimistic flush logic + that will attempt this flush. + +commit 04e0d0a5e77e4f3da6413e209ab8fd360dfcb899 +Merge: 37c4ddd1291 509729f9c1a +Author: Tal Levy +Date: Wed Jun 13 14:15:28 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 509729f9c1a5c91dcbed6906990852a7e28522c9 +Author: lcawl +Date: Wed Jun 13 13:37:35 2018 -0700 + + [DOCS] Shortens ML API intros + +commit 93662c63855aa72ceb241acf2d4241bb02266cad +Author: Costin Leau +Date: Wed Jun 13 23:37:51 2018 +0300 + + Use quotes in the call invocation (#31249) + + Adding quotes around call invocation as paths can contain spaces that + otherwise would cause the command to fail + +commit bbb7889236324d4890e7b3a316cac59519473857 +Author: Tal Levy +Date: Wed Jun 13 13:35:41 2018 -0700 + + move security ingest processors to a sub ingest directory (#31306) + + It makes sense to introduce new Security ingest + processors (example: #31087), and this change would + give them a good place to be written. + +commit af58dc56fefd88dba48d4fcca3246d468dd68bc9 +Author: Adrien Grand +Date: Wed Jun 13 22:21:09 2018 +0200 + + Add 5.6.11 version constant. + +commit 2c17ab7d2a9816a3cc9cc5f6d9adcfa21e396e71 +Author: Adrien Grand +Date: Wed Jun 13 22:06:43 2018 +0200 + + Fix version detection. + +commit 43cb24035e4267e22041732bb805a07be47760a8 +Author: Costin Leau +Date: Wed Jun 13 23:08:18 2018 +0300 + + SQL: Whitelist SQL utility class for better scripting (#30681) + + Add SQL class for reusing code inside SQL functions within Painless + + Fix #29832 + +commit d4262de83a7a3fffd35ebcdc73c1014fbac8f19d +Author: Zachary Tong +Date: Wed Jun 13 15:42:20 2018 -0400 + + [Docs] All Rollup docs experimental, agg limitations, clarify DeleteJob (#31299) + + - All rollup pages should be marked as experimental instead of just + the top page + - While the job config docs state which aggregations are allowed, adding + a section which specifically details this in one place is more convenient + for the user + - Add a clarification that the DeleteJob API does not delete the rollup + data, just the rollup job. + +commit 37c4ddd1291ffed2cd4539bb739da1c784ede824 +Merge: 6c9e0aeca36 664903a70ad +Author: Tal Levy +Date: Wed Jun 13 12:13:05 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 664903a70ad00ec6400f9bfccb620517de355e77 +Author: Luca Cavanna +Date: Wed Jun 13 20:37:12 2018 +0200 + + CCS: don't proxy requests for already connected node (#31273) + + Cross-cluster search selects a subset of nodes for each remote cluster + and sends requests only to them, which will act as a proxy and properly + redirect such requests to the target nodes that hold the relevant data. + What happens today is that every time we send a request to a remote + cluster, it will be sent to the next node in the proxy list + (in round-robin fashion), regardless of whether the target node is + already amongst the ones that we are connected to. In case for instance + we need to send a shard search request to a data node that's also one of + the selected proxy nodes, we may end up sending the request to it + through one of the other proxy nodes. + This commit optimizes this case to make sure that whenever we are + already connected to a remote node, we will send a direct request rather + than using the next proxy node. + There is a side-effect to this, which is that round-robin will be a bit + unbalanced as the data nodes that are also selected as proxies will + receive more requests. + +commit 018d3fc81fed01365bd5ac0bf39764fef75b1e4b +Author: Igor Motov +Date: Wed Jun 13 14:11:48 2018 -0400 + + Mute ScriptedMetricAggregatorTests testSelfReferencingAggStateAfterMap + + Tracked by #31307 + +commit 01d64b128b214c2ac5afc1b0b514dd0d45e60c1b +Author: Andy Bristol +Date: Wed Jun 13 11:15:04 2018 -0700 + + [test] opensuse packaging turn up debug logging + +commit 73742a4be90b20ffa81755d9ef06cc8d989fba65 +Author: Dimitris Athanasiou +Date: Wed Jun 13 17:59:43 2018 +0100 + + Add unreleased version 6.3.1 + +commit 58e9446e00b89270eb099c3cf35e81acbe566d05 +Author: Colin Goodheart-Smithe +Date: Wed Jun 13 17:24:32 2018 +0100 + + Removes experimental tag from scripted_metric aggregation (#31298) + +commit 6c9e0aeca36b024756601c91fa0ea815b2497df2 +Author: Colin Goodheart-Smithe +Date: Wed Jun 13 17:17:32 2018 +0100 + + Adds API to assign or change the policy for an index (#31277) + + * Adds API to assign or change the policy for an index + + This api will change `index.lifecycle.name` for all indexes in the + provided pattern as long as they are not currently in the shrink action. + + Later changes will loosen this restriction so an index is only reject + if it is currently in the shrink action AND the diff between the old + and new policy includes changes to the shrink action. + + Also later changes will detect if the current step is not present in + the new policy and move the index onto the next available step + + * Changes name to SetPolicyForIndexAction + + Also changes all related Classes and api endpoints + + * fixes tests + +commit 6785391a5f74dac2509610cf547c93dceb9c5ac7 +Merge: 43eaec304be a486177a191 +Author: Colin Goodheart-Smithe +Date: Wed Jun 13 16:38:35 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit a486177a19171ab408cb88c1b1ef8d75ef2427fd +Author: Zachary Tong +Date: Wed Jun 13 11:31:04 2018 -0400 + + [Rollup] Metric config parser must use builder so validation runs (#31159) + + The parser for the Metric config was directly instantiating + the config object, rather than using the builder. That means it was + bypassing the validation logic built into the builder, and would allow + users to create invalid metric configs (like using unsupported metrics). + + The job would later blow up and abort due to bad configs, but this isn't + immediately obvious to the user since the PutJob API succeeded. + +commit 43eaec304bed680957f014c7fb39e2668391fa75 +Author: Colin Goodheart-Smithe +Date: Wed Jun 13 16:15:26 2018 +0100 + + Fixes ExplainLifecycleRequestTests + +commit 88f44a9f66b61fb158b3f4605a57909c86037489 +Author: David Kyle +Date: Wed Jun 13 15:42:18 2018 +0100 + + [ML] Check licence when datafeeds use cross cluster search (#31247) + + This change prevents a datafeed using cross cluster search from starting if the remote cluster + does not have x-pack installed and a sufficient license. The check is made only when starting a + datafeed. + +commit 7199d5f0e680182bdabba80c9976c6b74f00bedd +Author: Jason Tedor +Date: Wed Jun 13 10:16:46 2018 -0400 + + Add notion of internal index settings (#31286) + + We have some use cases for an index setting to only be manageable by + dedicated APIs rather than be updateable via the update settings + API. This commit adds the notion of an internal index setting. Such + settings can be set on create index requests, they can not be changed + via the update settings API, yet they can be changed by action on behalf + of or triggered by the user via dedicated APIs. + +commit 77bb93557eabc3728a6a5362d9be365032f55870 +Author: Nik Everett +Date: Wed Jun 13 09:33:06 2018 -0400 + + Test: Remove broken yml test feature (#31255) + + The `requires_replica` yaml test feature hasn't worked for years. This + is what happens if you try to use it: + ``` + > Throwable #1: java.lang.NullPointerException + > at __randomizedtesting.SeedInfo.seed([E6602FB306244B12:6E341069A8D826EA]:0) + > at org.elasticsearch.test.rest.yaml.Features.areAllSupported(Features.java:58) + > at org.elasticsearch.test.rest.yaml.section.SkipSection.skip(SkipSection.java:144) + > at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:321) + ``` + + None of our tests use it. + +commit 24163d10b7639ea8f66ec5fbf52a80ab90b0c356 +Author: Luca Cavanna +Date: Wed Jun 13 15:06:13 2018 +0200 + + REST hl client: cluster health to default to cluster level (#31268) + + With #29331 we added support for the cluster health API to the + high-level REST client. The transport client does not support the level + parameter, and it always returns all the info needed for shards level + rendering. We have maintained that behaviour when adding support for + cluster health to the high-level REST client, to ease migration, but the + correct thing to do is to default the high-level REST client to + `cluster` level, which is the same default as when going through the + Elasticsearch REST layer. + +commit 9fd4f3f8545cc9b25dceea0e4cc355c06e708244 +Merge: 2fae7a2ff5e 66f7dd2c4d2 +Author: Colin Goodheart-Smithe +Date: Wed Jun 13 13:41:17 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 66f7dd2c4d2b3400f196228d730d92b7b522bde6 +Author: Tom Veasey +Date: Wed Jun 13 13:12:53 2018 +0100 + + [ML] Update test thresholds to account for changes to memory control (#31289) + + To avoid temporary failures, this also disables these tests until elastic/ml-cpp#122 is committed. + +commit 8c9360b5a1356a5c3f5d8ee2be9bd0cdf552216b +Author: Boaz Leskes +Date: Wed Jun 13 13:22:34 2018 +0200 + + Log warnings when cluster state publication failed to some nodes (#31233) + + If the publishing of a cluster state to a node fails, we currently only log it as debug information and + only on the master. This makes it hard to see the cause of (test) failures when logging is set to + default levels. This PR adds a warn level log on the node receiving the cluster state when it fails to + deserialise the cluster state and a warn level log on the master with a list of nodes for which + publication failed. + +commit 8b4d80ad09261d5db2595ded3ef68032c6898414 +Author: Tanguy Leroux +Date: Wed Jun 13 12:40:22 2018 +0200 + + Fix AntFixture waiting condition (#31272) + + The AntFixture waiting condition is evaluated to false + but it should be true. + +commit 489db54e5745dc375bdb399aa3e7c37783a6b9ec +Author: David Turner +Date: Wed Jun 13 11:25:26 2018 +0100 + + Ignore numeric shard count if waiting for ALL (#31265) + + Today, if GET /_cluster/health?wait_for_active_shards=all does not immediately + succeed then it throws an exception due to an erroneous and unnecessary call to + ActiveShardCount#enoughShardsActive(). This commit fixes this logic. + + Fixes #31151 + +commit 5c77ebe89d2c0246c0c505f8e9466c69490eebe5 +Author: Dimitris Athanasiou +Date: Wed Jun 13 11:20:38 2018 +0100 + + [ML] Implement new rules design (#31110) + + Rules allow users to supply a detector with domain + knowledge that can improve the quality of the results. + The model detects statistically anomalous results but it + has no knowledge of the meaning of the values being modelled. + + For example, a detector that performs a population analysis + over IP addresses could benefit from a list of IP addresses + that the user knows to be safe. Then anomalous results for + those IP addresses will not be created and will not affect + the quantiles either. + + Another example would be a detector looking for anomalies + in the median value of CPU utilization. A user might want + to inform the detector that any results where the actual + value is less than 5 is not interesting. + + This commit introduces a `custom_rules` field to the `Detector`. + A detector may have multiple rules which are combined with `or`. + + A rule has 3 fields: `actions`, `scope` and `conditions`. + + Actions is a list of what should happen when the rule applies. + The current options include `skip_result` and `skip_model_update`. + The default value for `actions` is the `skip_result` action. + + Scope is optional and allows for applying filters on any of the + partition/over/by field. When not defined the rule applies to + all series. The `filter_id` needs to be specified to match the id + of the filter to be used. Optionally, the `filter_type` can be specified + as either `include` (default) or `exclude`. When set to `include` + the rule applies to entities that are in the filter. When set to + `exclude` the rule only applies to entities not in the filter. + + There may be zero or more conditions. A condition requires `applies_to`, + `operator` and `value` to be specified. The `applies_to` value can be + either `actual`, `typical` or `diff_from_typical` and it specifies + the numerical value to which the condition applies. The `operator` + (`lt`, `lte`, `gt`, `gte`) and `value` complete the definition. + Conditions are combined with `and` and allow to specify numerical + conditions for when a rule applies. + + A rule must either have a scope or one or more conditions. Finally, + a rule with scope and conditions applies when all of them apply. + +commit eb85852a188492bfc8da51ea34bfa1aaa28b4868 +Author: Alan Woodward +Date: Wed Jun 13 10:53:18 2018 +0100 + + index_prefixes back-compat should test 6.3 (#30951) + +commit a65b18f19dd90c83dde1d4156141a693ce754c2c +Author: Ryan Ernst +Date: Wed Jun 13 09:58:13 2018 +0200 + + Core: Remove plain execute method on TransportAction (#30998) + + TransportAction has many variants of execute. One of those variants + executes by returning a future, which is then often blocked on by + calling get(). This commit removes this variant of execute, instead + using a helper method for tests that want to block, or having tests + pass in a PlainActionFuture directly as a listener. + + Co-authored-by: Simon Willnauer + +commit 1f6e87400258fdce770a4e64ed817e1cfd2a643f +Author: Tanguy Leroux +Date: Wed Jun 13 09:22:17 2018 +0200 + + Update checkstyle to 8.10.1 (#31269) + +commit 16d593b22f94e94332aa19363cac4cdecbec7149 +Author: Martijn van Groningen +Date: Wed Jun 13 07:25:19 2018 +0200 + + Set analyzer version in PreBuiltAnalyzerProviderFactory (#31202) + + instead of lamda that creates the analyzer + +commit 56ffe553e59b8af9337ed162226a95bf10d037df +Author: Tim Brooks +Date: Tue Jun 12 23:15:24 2018 -0600 + + Modify pipelining handlers to require full requests (#31280) + + Currently the http pipelining handlers seem to support chunked http + content. However, this does not make sense. There is a content + aggregator in the pipeline before the pipelining handler. This means the + pipelining handler should only see full http messages. Additionally, the + request handler immediately after the pipelining handler only supports + full messages. + + This commit modifies both nio and netty4 pipelining handlers to assert + that an inbound message is a full http message. Additionally it removes + the tests for chunked content. + +commit 0bfd18cc8b6ab7e58d1d34c50ae7a3b441799761 +Author: Jason Tedor +Date: Tue Jun 12 19:26:18 2018 -0400 + + Revert upgrade to Netty 4.1.25.Final (#31282) + + This reverts upgrading to Netty 4.1.25.Final until we have a cleaner + solution to dealing with the object cleaner thread. + +commit 905663a94264aa43a772e11eade16c9262cf546c +Author: Jason Tedor +Date: Tue Jun 12 19:13:02 2018 -0400 + + Use armored input stream for reading public key (#31229) + + This was silly; Bouncy Castle has an armored input stream for reading + keys in ASCII armor format. This means that we do not need to strip the + header ourselves and base64 decode the key. This had problems anyway + because of discrepancies in the padding that Bouncy Castle would produce + and the JDK base64 decoder was expecting. Now that we armor input/output + the whole way during tests, we fix all random failures in test cases + too. + +commit 2fae7a2ff5ed863a1145c40a6595884d0778bc3b +Author: Tal Levy +Date: Tue Jun 12 14:47:39 2018 -0700 + + fix explainlifecyclerequesttest + +commit f88b9e83cfdac7010736fd221282f1986d3b1b39 +Author: James Baiera +Date: Tue Jun 12 14:58:53 2018 -0400 + + Fix Netty 4 Server Transport tests. Again. + +commit bd51b6b96a36ffa3e04c81ab66032a61983d76c4 +Author: Luca Cavanna +Date: Tue Jun 12 21:00:33 2018 +0200 + + REST hl client: adjust wait_for_active_shards param in cluster health (#31266) + + The default wait_for_active_shards is NONE for cluster health, which + differs from all the other API in master, hence we need to make sure to + set the parameter whenever it differs from NONE (0). The test around + this also had a bug, which is why this was not originally uncovered. + + Relates to #29331 + +commit 92eb324776f26e0edf8b7957c81950890b8ce4d9 +Author: Luca Cavanna +Date: Tue Jun 12 21:00:06 2018 +0200 + + REST high-level Client: remove deprecated API methods (#31200) + + This commit removes all the API methods that accept a `Header` varargs + argument, in favour of the newly introduced API methods that accept a + `RequestOptions` argument. + + Relates to #31069 + +commit 59ea407333dd5a0b7f1971e66c1d58efacea3834 +Merge: ff80202353e bd81b95e953 +Author: Tal Levy +Date: Tue Jun 12 10:54:58 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit bd81b95e953a876b60aa463e6876d50e5da00df1 +Author: Costin Leau +Date: Tue Jun 12 19:46:08 2018 +0300 + + [DOCS] Mark SQL feature as experimental + +commit ff0f4ece2f8a58eef934f8eb632cd9b863be2fb2 +Author: Lisa Cawley +Date: Tue Jun 12 08:57:11 2018 -0700 + + [DOCS] Updates machine learning custom URL screenshots (#31222) + +commit 666997830e47a3c39da1da4a9e11426fda85429a +Author: Jason Tedor +Date: Tue Jun 12 11:34:40 2018 -0400 + + Fix naming conventions check for XPackTestCase + + This class needs to be abstract or it fails the naming convention check. + +commit f738b7d9f8d2c936a4876b7eba6243c8442099da +Author: Jason Tedor +Date: Tue Jun 12 11:31:43 2018 -0400 + + Fix security Netty 4 transport tests + + This test suite needs to filter out the object cleaner thread too so + this commit does that. + +commit a36543531b4f547bbb7be8156d4c1e55f0d53cf9 +Author: Jason Tedor +Date: Tue Jun 12 10:17:41 2018 -0400 + + Fix race in clear scroll (#31259) + + Here is the problem: if two threads are racing and one hits a failure + freeing a context and the other succeeded, we can expose the value of + the has failure marker to the succeeding thread before the failing + thread has had a chance to set the failure marker. This is a problem if + the failing thread counted down the expected number of operations, then + be put to sleep by a gentle lullaby from the OS, and then the other + thread could count down to zero. Since the failing thread did not get to + set the failure marker, the succeeding thread would respond that the + clear scroll succeeded and that makes that thread a liar. This commit + addresses by first setting the failure marker before we potentially + expose its value to another thread. + +commit e988ace5f75ac672667f37170634fd6b775e590a +Author: Albert Zaharovits +Date: Tue Jun 12 16:19:43 2018 +0300 + + [DOCS] Clarify audit index settings when remote indexing (#30923) + +commit ff80202353eaa2dae09504c0e304824b913c3614 +Merge: 9a9c13ede0f ef453b31b43 +Author: Colin Goodheart-Smithe +Date: Tue Jun 12 12:52:04 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit ef453b31b43f9e6132def1461e43654ffb9a0be2 +Author: Yuri Tceretian +Date: Tue Jun 12 14:45:27 2018 +0300 + + Delete typos in SAML docs (#31199) + + * Delete typos in SAML docs saml-guide.asciidoc + +commit d5e8a5cd6910ac75042cffc64012b506933490f1 +Author: Van0SS +Date: Tue Jun 12 07:34:06 2018 -0400 + + REST high-level client: add Cluster Health API (#29331) + + Relates to #27205 + +commit 5f84e18c7296e8ccfd1405fd768887cc16f96f7e +Author: Dimitris Athanasiou +Date: Tue Jun 12 11:36:26 2018 +0100 + + [ML][TEST] Mute tests using rules (#31204) + + This is in preparation of pushing the new + rules design in the `ml-cpp` side. These + tests will be switched on again after merging + in the new rules implementation. + +commit b2e48c9fa7b67cb7bbea0d72500332baa2459b92 +Author: Ioannis Kakavas +Date: Tue Jun 12 12:23:40 2018 +0300 + + Support RequestedAuthnContext (#31238) + + * Support RequestedAuthnContext + + This implements limited support for RequestedAuthnContext by : + - Allowing SP administrators to define a list of authnContextClassRef + to be included in the RequestedAuthnContext of a SAML Authn Request + - Veirifying that the authnContext in the incoming SAML Asertion's + AuthnStatement contains one of the requested authnContextClassRef + - Only EXACT comparison is supported as the semantics of validating + the incoming authnContextClassRef are deployment dependant and + require pre-established rules for MINIMUM, MAXIMUM and BETTER + + Also adds necessary AuthnStatement validation as indicated by [1] and + [2] + + [1] https://docs.oasis-open.org/security/saml/v2.0/saml-core-2.0-os.pdf + 3.4.1.4, line 2250-2253 + [2] https://kantarainitiative.github.io/SAMLprofiles/saml2int.html + [SDP-IDP10] + +commit b4469f82b9d6fcb2197e38d6610b229d3b2b993a +Author: Simon Willnauer +Date: Tue Jun 12 10:34:06 2018 +0200 + + Ensure LuceneChangesSnapshot reads in leaf order (#31246) + + Today we re-initialize DV instances while we read docs + for the snapshot. This is caused by the fact that we sort the + docs by seqID which causes then to be our of order. This change + sorts documents temporarily by docID, fetches the metadata (not source) + into a in-memory datastructure and sorts it back. + This allows efficient reuse of DV instances. + +commit 113c1916eeffb8fb453d56c97942eefa19bcb087 +Author: Luca Cavanna +Date: Tue Jun 12 10:27:51 2018 +0200 + + SyncedFlushResponse to implement ToXContentObject (#31155) + + The response currently implements ToXContentFragment although the only time it's used + it is supposed to print out a complete object rather than a fragment. Note that this + is the client version of the response, used only in the high-level client. + +commit 7d7ead95b2a3b967f788f1fa56f118ee77d857fe +Author: olcbean <26058559+olcbean@users.noreply.github.com> +Date: Tue Jun 12 10:26:17 2018 +0200 + + Add Get Aliases API to the high-level REST client (#28799) + + Given the weirdness of the response returned by the get alias API, we went for a client specific response, which allows us to hold the error message, exception and status returned as part of the response together with aliases. See #30536 . + + Relates to #27205 + +commit a178290b60e58dd5a0ca90efa5b0443de101038b +Author: Christoph Büscher +Date: Tue Jun 12 10:03:51 2018 +0200 + + Remove some line length supressions (#31209) + +commit 72f8199279a3fa6a053fbce5d11a68ee6127591d +Author: Aditya Dhulipala +Date: Tue Jun 12 00:53:36 2018 -0700 + + Validate xContentType in PutWatchRequest. (#31088) + + Trying to post a new watch without any body currently results in a + NullPointerException. This change fixes that by validating that + Post and Put requests always have a body. + + Closes #30057 + +commit 6030d4be1ecf5cccf3bdc019a288f3b097304428 +Author: Martijn van Groningen +Date: Tue Jun 12 07:49:03 2018 +0200 + + [INGEST] Interrupt the current thread if evaluation grok expressions take too long (#31024) + + This adds a thread interrupter that allows us to encapsulate calls to org.joni.Matcher#search() + This method can hang forever if the regex expression is too complex. + + The thread interrupter in the background checks every 3 seconds whether there are threads + execution the org.joni.Matcher#search() method for longer than 5 seconds and + if so interrupts these threads. + + Joni has checks that that for every 30k iterations it checks if the current thread is interrupted and + if so returns org.joni.Matcher#INTERRUPTED + + Closes #28731 + +commit 9a9c13ede0f9c7fe101829933f2591f55cc2aa78 +Merge: 0541684df3a 1dbe554e5e9 +Author: Tal Levy +Date: Mon Jun 11 22:22:59 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 1dbe554e5e9b6b0cefa8cf6a0a9cbc547fdf511c +Author: Jason Tedor +Date: Mon Jun 11 22:19:19 2018 -0400 + + Suppress extras FS on caching directory tests + + This filesystem needs to be suppressed during these tests because it + adds random files to the directory upon directory creation. That means + that the size of these directories is off from what we expect them to + be. Rather than loosening the assertion which could hide bugs on real + directories, this commit suppresses this file system in this test suite. + +commit 32cbbc2ecfbb28ebd02c5a09f9b21b372d906464 +Author: Jason Tedor +Date: Mon Jun 11 22:04:36 2018 -0400 + + Revert "[DOCS] Added 6.3 info & updated the upgrade table. (#30940)" + + This reverts commit 40d31d2ae0e5d1004a72de44738948434c8e7a06. + +commit 0e6589f48996d7bc13ea579fd628947181d3ab73 +Author: Jason Tedor +Date: Mon Jun 11 22:04:30 2018 -0400 + + Revert "Fix snippets in upgrade docs" + + This reverts commit 955542a50a4405c54fc77f733696088638334661. + +commit 955542a50a4405c54fc77f733696088638334661 +Author: Jason Tedor +Date: Mon Jun 11 21:32:39 2018 -0400 + + Fix snippets in upgrade docs + + The upgrade docs were using the json language tag but console wants + these to be js for proper formatting. This commit addresses that. + +commit 0541684df3a49682182d6fcd00df3b394e151046 +Merge: b865d0476d6 40d31d2ae0e +Author: Tal Levy +Date: Mon Jun 11 17:20:05 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 40d31d2ae0e5d1004a72de44738948434c8e7a06 +Author: debadair +Date: Mon Jun 11 15:53:25 2018 -0700 + + [DOCS] Added 6.3 info & updated the upgrade table. (#30940) + +commit 0d9b78834f028824bdd61e7a8c4a90033eaae54b +Author: Nik Everett +Date: Mon Jun 11 17:07:27 2018 -0400 + + LLClient: Support host selection (#30523) + + Allows users of the Low Level REST client to specify which hosts a + request should be run on. They implement the `NodeSelector` interface + or reuse a built in selector like `NOT_MASTER_ONLY` to chose which nodes + are valid. Using it looks like: + ``` + Request request = new Request("POST", "/foo/_search"); + RequestOptions options = request.getOptions().toBuilder(); + options.setNodeSelector(NodeSelector.NOT_MASTER_ONLY); + request.setOptions(options); + ... + ``` + + This introduces a new `Node` object which contains a `HttpHost` and the + metadata about the host. At this point that metadata is just `version` + and `roles` but I plan to add node attributes in a followup. The + canonical way to **get** this metadata is to use the `Sniffer` to pull + the information from the Elasticsearch cluster. + + I've marked this as "breaking-java" because it breaks custom + implementations of `HostsSniffer` by renaming the interface to + `NodesSniffer` and by changing it from returning a `List` to a + `List`. It *shouldn't* break anyone else though. + + Because we expect to find it useful, this also implements `host_selector` + support to `do` statements in the yaml tests. Using it looks a little + like: + + ``` + --- + "example test": + - skip: + features: host_selector + - do: + host_selector: + version: " - 7.0.0" # same syntax as skip + apiname: + something: true + ``` + + The `do` section parses the `version` string into a host selector that + uses the same version comparison logic as the `skip` section. When the + `do` section is executed it passed the off to the `RestClient`, using + the `ElasticsearchHostsSniffer` to sniff the required metadata. + + The idea is to use this in mixed version tests to target a specific + version of Elasticsearch so we can be sure about the deprecation + logging though we don't currently have any examples that need it. We do, + however, have at least one open pull request that requires something + like this to properly test it. + + Closes #21888 + +commit 563141c6c9e642be572796aa2dd29dc04f9ba901 +Author: Jason Tedor +Date: Mon Jun 11 16:55:07 2018 -0400 + + Upgrade to Netty 4.1.25.Final (#31232) + + This commit upgrades us to Netty 4.1.25. This upgrade is more + challenging than past upgrades, all because of a new object cleaner + thread that they have added. This thread requires an additional security + permission (set context class loader, needed to avoid leaks in certain + scenarios). Additionally, there is not a clean way to shutdown this + thread which means that the thread can fail thread leak control during + tests. As such, we have to filter this thread from thread leak control. + +commit cb952bd9eca06e26e945c59e00eb2ee33719f6a9 +Author: Jason Tedor +Date: Mon Jun 11 16:53:40 2018 -0400 + + Enable custom credentials for core REST tests (#31235) + + The core REST tests with security currently use a hardcoded username and + password. This is not amenable to running these tests in scenarios where + the user controls the creation of the cluster and owns the credentials + for this cluster. This commit enables running the core REST tests with + security with a custom username and password. + +commit dda56fc0fc7ab4b76e32a80a57bd89cdf11a8aac +Author: Nhat Nguyen +Date: Mon Jun 11 12:47:38 2018 -0400 + + Move ESIndexLevelReplicationTestCase to test framework (#31243) + + Other components might benefit from the testing infra provided by + ESIndexLevelReplicationTestCase. This commit moves it to the test + framework. + +commit c064b507dfa8259d6eaef36f26259941579a851f +Author: Lee Hinman +Date: Mon Jun 11 09:44:50 2018 -0600 + + Encapsulate Translog in Engine (#31220) + + This removes the abstract `getTranslog` method in `Engine`, instead leaving it + to the abstract implementations of the other methods that use the translog. This + allows future Engines not to have a Translog, as instead they must implement the + methods that use the translog pieces to return necessary values. + +commit 99e04582ded791e87ffa372962407ba34bdbd228 +Author: Nhat Nguyen +Date: Mon Jun 11 11:06:28 2018 -0400 + + HLRest: Add get index templates API (#31161) + + Relates #27205 + +commit bf5866048262d897ede629293b6fe26c3b108b7a +Author: Tanguy Leroux +Date: Mon Jun 11 15:12:12 2018 +0200 + + Remove all unused imports and fix CRLF (#31207) + + The X-Pack opening and the recent other refactorings left a lot of + unused imports in the codebase. This commit removes them all. + +commit a1916658a95904e68499c65c84ac2adb85f72732 +Author: Tanguy Leroux +Date: Mon Jun 11 12:45:27 2018 +0200 + + [Tests] Fix self-referencing tests + + This commit adapts some test after #31044 has been merged. + +commit b865d0476d6acc293c37ac6a3ef7d139937d7909 +Author: Colin Goodheart-Smithe +Date: Mon Jun 11 10:18:43 2018 +0100 + + Makes IndexLifecycleMetadata feature aware + +commit f9e8afd357bc564429580eaec573296540977622 +Author: Yannick Welsch +Date: Mon Jun 11 11:09:45 2018 +0200 + + [TEST] Fix testRecoveryAfterPrimaryPromotion + + This test was failing from time to time due to a ConcurrentModificationException, which + was triggered due to the primary-replica resync running concurrently with shards being + removed. + + Closes #30767 + +commit 9ee492a3f0768a3790671bfe46223e700082730a +Merge: 6b6882900d5 42608881b09 +Author: Colin Goodheart-Smithe +Date: Mon Jun 11 09:47:26 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 42608881b0912d3d71c4d8dc5641df73d9ccc088 +Author: Tanguy Leroux +Date: Mon Jun 11 09:32:12 2018 +0200 + + [Docs] Remove mention pattern files in Grok processor (#31170) + + Pattern files have been removed in + 16fa3e546e172d3d194c2711654824d4851d69f8 + +commit 467ea50c4e119ce8ea3d4d4d7ed3ce5f8d9b7148 +Author: Yannick Welsch +Date: Mon Jun 11 09:12:44 2018 +0200 + + Use stronger write-once semantics for Azure repository (#30437) + + There's no need for an extra blobExists() call when writing a blob to the Azure service. Azure + provides an option (with stronger consistency guarantees) on the upload method that guarantees + that the blob that's uploaded does not already exist. This saves one network roundtrip. + + Relates to #19749 + +commit 4e9b5549486f53fc95d6e40299c87ed3f581adbe +Author: Yannick Welsch +Date: Mon Jun 11 09:09:23 2018 +0200 + + Don't swallow exceptions on replication (#31179) + + Swallowing these exceptions is dangerous as they can result in replicas going out-of-sync with the primary. + + Follow-up to #28571 + +commit f825a530b8aab70ef3efc0aee31495c958dca616 +Author: Simon Willnauer +Date: Mon Jun 11 08:49:18 2018 +0200 + + Limit the number of concurrent requests per node (#31206) + + With `max_concurrent_shard_requests` we used to throttle / limit + the number of concurrent shard requests a high level search request + can execute per node. This had several problems since it limited the + number on a global level based on the number of nodes. This change + now throttles the number of concurrent requests per node while still + allowing concurrency across multiple nodes. + + Closes #31192 + +commit 85c26d682a5a6fbaf345d66b9a783b78f4b1cc63 +Author: rationull +Date: Sun Jun 10 23:39:05 2018 -0700 + + Call ensureNoSelfReferences() on _agg state variable after scripted metric agg script executions (#31044) + + Previously this was called for the combine script only. This change checks for self references for + init, map, and reduce scripts as well, and adds unit test coverage for the init, map, and combine cases. + +commit bd5c1a4590ea72cecf301ac85244e5249a5af269 +Author: Ryan Ernst +Date: Mon Jun 11 02:03:21 2018 +0200 + + Move java version checker back to its own jar (#30708) + + The java version checker requires being written with java 7 APIs. + In order to use java 8 apis in other launcher utilities, this commit + moves the java version checker back to its own jar. + +commit f8cbc812585092112660ae888e9d23609992d6ac +Author: Andy Bristol +Date: Sun Jun 10 08:12:46 2018 -0700 + + [test] add fix for rare virtualbox error (#31212) + + See the vagrant issue mentioned in this commit for details. This error + has happened a couple times in packaging test CI builds with workers + using virtualbox 5.2.10r122088 + +commit 9feff9809bb4279ed638c88b83edaca3ec2fd2d7 +Author: Nhat Nguyen +Date: Sat Jun 9 20:04:45 2018 -0400 + + CCR side of #31213 + + Relates #31213 + +commit afe02a3cac3e0cd81abd3ec6e7716e345cc6b8ba +Merge: 64b4cdeda64 aa8aa0d9e02 +Author: Nhat Nguyen +Date: Sat Jun 9 19:53:21 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Move default location of dependencies report (#31228) + Remove dependencies report task dependencies (#31227) + Add recognition of MPL 2.0 (#31226) + Fix unknown licenses (#31223) + Remove version from license file name for GCS SDK (#31221) + Fully encapsulate LocalCheckpointTracker inside of the engine (#31213) + [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 (#31160) + Add licenses for transport-nio (#31218) + Remove DocumentFieldMappers#simpleMatchToFullName. (#31041) + Allow to trim all ops above a certain seq# with a term lower than X, post backport fix (#31211) + Compliant SAML Response destination check (#31175) + Remove DocumentFieldMappers#smartNameFieldMapper, as it is no longer needed. (#31018) + Remove extraneous references to 'tokenized' in the mapper code. (#31010) + Allow to trim all ops above a certain seq# with a term lower than X (#30176) + SQL: Make a single JDBC driver jar (#31012) + Enhance license detection for various licenses (#31198) + [DOCS] Add note about long-lived idle connections (#30990) + Move number of language analyzers to analysis-common module (#31143) + Default max concurrent search req. numNodes * 5 (#31171) + flush job to ensure all results have been written (#31187) + +commit aa8aa0d9e0275ba30c58cc58b14ca39adcf7eff7 +Author: Jason Tedor +Date: Sat Jun 9 09:50:36 2018 -0400 + + Move default location of dependencies report (#31228) + + This commit moves the default location of the full dependencies report + to be under the reports directory to align it with the location for the + dependenciesInfo task output. + +commit 1d840f934824641380395095d41dcf0671e93764 +Author: Jason Tedor +Date: Sat Jun 9 09:50:24 2018 -0400 + + Remove dependencies report task dependencies (#31227) + + A previous commit tried to add task dependencies for the + :distribution:generateDependenciesReport task so that a user did not + have to run "dependenciesInfo + :distribution:generateDependenciesReport". However this method did not + reliably add all task dependencies due to task ordering issues in + previous versions of Gradle and our build. This commit removes this for + now and a user will continue to have to run "dependenciesInfo + :distribution:generateDependenciesReport". + +commit 1082ea57c819301afff2e6cb0192c552112db4ff +Author: Jason Tedor +Date: Sat Jun 9 09:46:57 2018 -0400 + + Add recognition of MPL 2.0 (#31226) + + This commit adds the ability for the dependencies info check to + recognize version 2.0 of the Mozilla Public License. + +commit 65c107b47d6afaae8851e07b3d239c9cabb709e3 +Author: Jason Tedor +Date: Sat Jun 9 07:28:41 2018 -0400 + + Fix unknown licenses (#31223) + + The goal of this commit is to address unknown licenses when producing + the dependencies info report. We have two different checks that we run + on licenses. The first check is whether or not we have stashed a copy of + the license text for a dependency in the repository. The second is to + map every dependency to a license type (e.g., BSD 3-clause). The problem + here is that the way we were handling licenses in the second check + differs from how we handle licenses in the first check. The first check + works by finding a license file with the name of the artifact followed + by the text -LICENSE.txt. Yet in some cases we allow mapping an artifact + name to another name used to check for the license (e.g., we map + lucene-.* to lucene, and opensaml-.* to shibboleth. The second check + understood the first way of looking for a license file but not the + second way. So in this commit we teach the second check about the + mappings from artifact names to license names. We do this by copying the + configuration from the dependencyLicenses task to the dependenciesInfo + task and then reusing the code from the first check in the second + check. There were some other challenges here though. For example, + dependenciesInfo was checking too many dependencies. For now, we should + only be checking direct dependencies and leaving transitive dependencies + from another org.elasticsearch artifact to that artifact (we want to do + this differently in a follow-up). We also want to disable + dependenciesInfo for projects that we do not publish, users only care + about licenses they might be exposed to if they use our assembled + products. With all of the changes in this commit we have eliminated all + unknown licenses. A follow-up will enforce that when we add a new + dependency it does not get mapped to unknown, these will be forbidden in + the future. Therefore, with this change and earlier changes are left + having no unknown licenses and two custom licenses; custom here means it + does not map to an SPDX license type. Those two licenses are xz and + ldapsdk. A future change will not allow additional custom licenses + unless they are explicitly whitelisted. This ensures that if a new + dependency is added it is mapped to an SPDX license or mapped to custom + because it does not have an SPDX license. + +commit 3db1fe7afe8e53081fa38867b76fe818a9094ede +Author: Jason Tedor +Date: Fri Jun 8 21:19:16 2018 -0400 + + Remove version from license file name for GCS SDK (#31221) + + Most of our license file names strip the version off the artifact name + when deducing the license filename. However, the version on the GCS SDK + (google-api-services-storage) does not match the usual format and + instead starts with a vee. This means that the license filename for this + license ended up carrying the version and we should not do that. This + commit adjusts the regex the deduces the license filename to account for + this case, and adjusts the google-api-services-storage license files + accordingly. + +commit bdb0fb25555381299d80fed7efcbcab97a9d172f +Author: Lee Hinman +Date: Fri Jun 8 17:19:41 2018 -0600 + + Fully encapsulate LocalCheckpointTracker inside of the engine (#31213) + + * Fully encapsulate LocalCheckpointTracker inside of the engine + + This makes the Engine interface not expose the `LocalCheckpointTracker`, instead + exposing the pieces needed (like retrieving the local checkpoint) as individual + methods. + +commit cdb486ae70053450e6a07ea45b5322741240e6c9 +Author: Sue Gallagher <36747279+Sue-Gallagher@users.noreply.github.com> +Date: Fri Jun 8 14:41:01 2018 -0700 + + [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 (#31160) + + * [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes 28008 + + * [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 + + * [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 + + * [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 + +commit f6d69c9d40ea401a2347a2dd6594ffdb1ae046b3 +Author: Jason Tedor +Date: Fri Jun 8 17:15:28 2018 -0400 + + Add licenses for transport-nio (#31218) + + We have a direct dependency on Netty here so we should be attaching the + licenses. This is needed for accurate dependency-license reporting. + +commit 00b0e1006320cf8d11d2aaf2f4233dab1bcc1be3 +Author: Julie Tibshirani +Date: Fri Jun 8 13:53:35 2018 -0700 + + Remove DocumentFieldMappers#simpleMatchToFullName. (#31041) + + * Remove DocumentFieldMappers#simpleMatchToFullName, as it is duplicative of MapperService#simpleMatchToIndexNames. + * Rename MapperService#simpleMatchToIndexNames -> simpleMatchToFullName for consistency. + * Simplify EsIntegTestCase#assertConcreteMappingsOnAll to accept concrete fields instead of wildcard patterns. + +commit 12fa0f437a3ea5e26a9d2783fae54622ddea4826 +Author: Vladimir Dolzhenko +Date: Fri Jun 8 12:11:20 2018 -0700 + + Allow to trim all ops above a certain seq# with a term lower than X, post backport fix (#31211) + +commit b26aae3915e0402ab70504cdc4b8b1bb2b8f10ae +Author: Ioannis Kakavas +Date: Fri Jun 8 20:36:31 2018 +0300 + + Compliant SAML Response destination check (#31175) + + Make SAML Response Destination check compliant + + Only validate the Destination element of an incoming SAML Response + if Destination is present and the SAML Response is signed. + The standard [1] - 3.5.5.2 and [2] - 3.2.2 does mention that the + Destination element is optional and should only be verified when + the SAML Response is signed. Some Identity Provider implementations + are known to not set a Destination XML Attribute in their SAML + responses when those are not signed, so this change also aims to + enhance interoperability. + + [1] https://docs.oasis-open.org/security/saml/v2.0/saml-bindings-2.0-os.pdf + [2] https://docs.oasis-open.org/security/saml/v2.0/saml-core-2.0-os.pdf + +commit 8f607071b6e34990d4e42f235d014d02cd680e8b +Author: Julie Tibshirani +Date: Fri Jun 8 09:24:09 2018 -0700 + + Remove DocumentFieldMappers#smartNameFieldMapper, as it is no longer needed. (#31018) + +commit 2378fa19b8cf31399434815d067765917825924a +Author: Julie Tibshirani +Date: Fri Jun 8 09:23:46 2018 -0700 + + Remove extraneous references to 'tokenized' in the mapper code. (#31010) + + These are likely left over from when there were three options for + the index mapping ('no', 'analyzed', 'not_analyzed'). + +commit a86c0f8c25fb146793c4992f2a18caffeecdfca6 +Author: Vladimir Dolzhenko +Date: Fri Jun 8 09:16:38 2018 -0700 + + Allow to trim all ops above a certain seq# with a term lower than X (#30176) + + Allow to trim all ops above a certain seq# with a term lower than X + + Relates to #10708 + +commit 01140a3ad83cd13b20968e20118c5e373975d888 +Author: Igor Motov +Date: Fri Jun 8 10:15:28 2018 -0400 + + SQL: Make a single JDBC driver jar (#31012) + + Replaces zip archive containing multiple jars with a single JDBC driver jar + that shades all external dependencies. + + Closes #29856 + +commit e467e67fd460af8e63d6ea01424fb377b0273181 +Author: Jason Tedor +Date: Fri Jun 8 08:55:10 2018 -0400 + + Enhance license detection for various licenses (#31198) + + This commit enhances the license detection that we have for various + licenses. Here we improve the detection for all licenses (especially the + Apache 2.0 License), the BSD 2-clause license, the MIT (with + attribution) license, and we add detection for the BSD 3-clause + license. One way that we achieved this improvement is by changing how + the license files are read so that rather than reading them as a + multi-line string which ended up represented as "[line1, line2, line3, + ...]" internally, we read the full bytes of the license text and replace + all whitespace with a single space so the license text is now loaded as + "line1 line2 line3". For the MIT license we add the actual license text + and remove the "MIT" string as not all copies of the license clearly + indicate that the text is the MIT license. We take a similar strategy + for the BSD-2 and BSD-3 clause licenses. With this change, we reduce the + number of "custom" licenses in the codebase from 31 to 2. The two + remaining appear to be truly custom licenses, not carrying licenses + identifiable by SPDX. A follow-up will address "unknown" licenses. + +commit 8d4f09f7f20dc044fec89b80deb8bdac32c09d62 +Author: David Turner +Date: Fri Jun 8 13:36:19 2018 +0100 + + [DOCS] Add note about long-lived idle connections (#30990) + + Clarify that we expect to have idle inter-node connections within the cluster, + and that the network needs to be configured not to disrupt these. + +commit 07a57cc1310f04ee0544b9222e737897e9e3ff99 +Author: Martijn van Groningen +Date: Fri Jun 8 08:58:46 2018 +0200 + + Move number of language analyzers to analysis-common module (#31143) + + The following analyzers were moved from server module to analysis-common module: + `snowball`, `arabic`, `armenian`, `basque`, `bengali`, `brazilian`, `bulgarian`, + `catalan`, `chinese`, `cjk`, `czech`, `danish`, `dutch`, `english`, `finnish`, + `french`, `galician` and `german`. + + Relates to #23658 + +commit 435a825a53dbabc93add9d400c1a8653e16b8aad +Author: Simon Willnauer +Date: Fri Jun 8 08:33:01 2018 +0200 + + Default max concurrent search req. numNodes * 5 (#31171) + + We moved to 1 shard by default which caused some issues in how many + concurrent shard requests we allow by default. For instance searching + a 5 shard index on a single node will now be executed serially per shard + while we want these cases to have a good concurrency out of the box. This + change moves to `numNodes * 5` which corresponds to the default we used to + have in the previous version. + + Relates to #30783 + Closes #30994 + +commit 253b9986810e3879fa00c6a370eb848b8bb18a60 +Author: Hendrik Muhs +Date: Fri Jun 8 07:51:45 2018 +0200 + + flush job to ensure all results have been written (#31187) + + flush ml job to ensure all results have been written + + fixes #31173 + +commit 64b4cdeda64c2d749881af4b1271b1677aa4b76c +Merge: 5c6711b8a44 d6a4c14e1bf +Author: Jason Tedor +Date: Thu Jun 7 23:33:42 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: (53 commits) + Painless: Restructure/Clean Up of Spec Documentation (#31013) + Update ignore_unmapped serialization after backport + Add back dropped substitution on merge + high level REST api: cancel task (#30745) + Enable engine factory to be pluggable (#31183) + Remove vestiges of animal sniffer (#31178) + Rename elasticsearch-nio to nio (#31186) + Rename elasticsearch-core to core (#31185) + Move cli sub-project out of server to libs (#31184) + [DOCS] Fixes broken link in auditing settings + QA: Better seed nodes for rolling restart + [DOCS] Moves ML content to stack-docs + [DOCS] Clarifies recommendation for audit index output type (#31146) + Add nio-transport as option for http smoke tests (#31162) + QA: Set better node names on rolling restart tests + Add support for ignore_unmapped to geo sort (#31153) + Share common parser in some AcknowledgedResponses (#31169) + Fix random failure on SearchQueryIT#testTermExpansionExceptionOnSpanFailure + Remove reference to multiple fields with one name (#31127) + Remove BlobContainer.move() method (#31100) + ... + +commit d6a4c14e1bfe23aaa8c3ff67a6b57f695672b330 +Author: Jack Conradson +Date: Thu Jun 7 17:11:56 2018 -0700 + + Painless: Restructure/Clean Up of Spec Documentation (#31013) + + Full restructure of the spec into new sections for operators, statements, scripts, functions, lambdas, and regexes. Split of operators into 6 sections, a table, reference, array, numeric, boolean, and general. Clean up of all operators sections. Sporadic clean up else where. + +commit 972dcbc0ad501a60d73c60bf76dbffd92b56ed06 +Author: Igor Motov +Date: Thu Jun 7 16:44:20 2018 -0400 + + Update ignore_unmapped serialization after backport + + Update the serialization version of ignore_unmapped flag after backport + to 6.4 + + Relates #31153 + +commit d49c85d2e867778adc28b226b6d4512619d85fd5 +Author: Jason Tedor +Date: Thu Jun 7 17:40:47 2018 -0400 + + Add back dropped substitution on merge + + This was dropped accidentally during merge conflict resolution. This + commit adds back the substitution for elasticsearch-cli. + +commit e82e5cc2e86c8a32255d7d791d4062823fe1521c +Author: Paul Sanwald +Date: Thu Jun 7 14:02:23 2018 -0700 + + high level REST api: cancel task (#30745) + + * Initial commit of rest high level exposure of cancel task + + * fix javadocs + + * address some code review comments + + * update branch to use tasks namespace instead of cluster + + * High-level client: list tasks failure to not lose nodeId + + This commit reworks testing for `ListTasksResponse` so that random + fields insertion can be tested and xcontent equivalence can be checked + too. Proper exclusions need to be configured, and failures need to be + tested separately. This helped finding a little problem, whenever there + is a node failure returned, the nodeId was lost as it was never printed + out as part of the exception toXContent. + + * added comment + + * merge from master + + * re-work CancelTasksResponseTests to separate XContent failure cases from non-failure cases + + * remove duplication of logic in parser creation + + * code review changes + + * refactor TasksClient to support RequestOptions + + * add tests for parent task id + + * address final PR review comments, mostly formatting and such + +commit e481b860a197e7a079e8435aef52ae08fbcc5b7e +Author: Jason Tedor +Date: Thu Jun 7 17:01:06 2018 -0400 + + Enable engine factory to be pluggable (#31183) + + This commit enables the engine factory to be pluggable based on index + settings used when creating the index service for an index. + +commit d8c0a39c150a9cbfa0862222ee80c03117cbb052 +Author: Jason Tedor +Date: Thu Jun 7 17:00:22 2018 -0400 + + Remove vestiges of animal sniffer (#31178) + + We no longer need animal sniffer because we use JDK functionality + (introduced in JDK 9) to target older versions of the JDK for + compilation. This functionality means that the JDK handles the problem + of ensuring that we do not use JDK APIs from the version that we are + compiling from that are not available in the version that we are + compiling to. A previous commit removed this for the REST client (where + we target JDK 7) but a few traces were left behind. + +commit 5296c11e4f1c1d0022cc1f2685ae51dcdf759d9c +Author: Jason Tedor +Date: Thu Jun 7 17:00:00 2018 -0400 + + Rename elasticsearch-nio to nio (#31186) + + This commit renames :libs:elasticsearch-nio to :libs:nio. + +commit 94be9b471f6959ca5664a4a143809630a908f78e +Author: Jason Tedor +Date: Thu Jun 7 16:50:21 2018 -0400 + + Rename elasticsearch-core to core (#31185) + + This commit renames :libs:elasticsearch-core to :libs:core. + +commit b32cbc1baa4dbf859cbde91bc7ce5c42929ffadb +Author: Jason Tedor +Date: Thu Jun 7 16:35:34 2018 -0400 + + Move cli sub-project out of server to libs (#31184) + + This commit moves the cli sub-project out of server to libs where it + makes more sense. + +commit 5dc9e87bad4b81a3afa99e4e5b36ce3a29f9a811 +Author: lcawl +Date: Thu Jun 7 10:48:21 2018 -0700 + + [DOCS] Fixes broken link in auditing settings + +commit dfcc939ef88636954e7ccabb2949a6b70c3554ce +Author: Nik Everett +Date: Thu Jun 7 13:05:36 2018 -0400 + + QA: Better seed nodes for rolling restart + + Use all running nodes as unicast seeds in the rolling restart tests to + avoid a race between pinging and the tests. Without this if the tests + are too fast then when a new node comes up and pings its single + configured seed node that node *might* not have a ping from the other + running node. + +commit 1de38a24882630fdb3ffd30499e74c592b671de5 +Author: lcawl +Date: Thu Jun 7 09:26:00 2018 -0700 + + [DOCS] Moves ML content to stack-docs + +commit d0f35d204e4a0d7806d5b92ca0caa17016d039e6 +Author: Lisa Cawley +Date: Thu Jun 7 08:55:14 2018 -0700 + + [DOCS] Clarifies recommendation for audit index output type (#31146) + +commit 237f9b89308568ff662a06e6e49d75708e2468ae +Author: Tim Brooks +Date: Thu Jun 7 09:46:36 2018 -0600 + + Add nio-transport as option for http smoke tests (#31162) + + This is related to #27260 and #28898. This commit adds the transport-nio + plugin as a random option when running the http smoke tests. As part of + this PR, I identified an issue where cors support was not properly + enabled causing these tests to fail when using transport-nio. This + commit also fixes that issue. + +commit 56207ea43deaa6e6bb25ad0f887ee2b029ce0e40 +Author: Nik Everett +Date: Thu Jun 7 11:19:34 2018 -0400 + + QA: Set better node names on rolling restart tests + + These should help with debugging failures. + +commit 7a9d9b0abfa09f816aeaa92c17bfe538b060e45d +Author: Igor Motov +Date: Thu Jun 7 11:11:13 2018 -0400 + + Add support for ignore_unmapped to geo sort (#31153) + + Adds support for `ignore_unmapped` parameter in geo distance sorting, + which is functionally equivalent to specifying an `unmapped_type` in + the field sort. + + Closes #28152 + +commit c352ff1615a3f6e5b4b04481dc6d33c13825e0d8 +Author: Christoph Büscher +Date: Thu Jun 7 13:52:10 2018 +0200 + + Share common parser in some AcknowledgedResponses (#31169) + + Several AcknowledgedResponse implementations only parse the boolean acknowledged + flag and then create an instance of their class using that flag. This can be + simplified by adding this basic parser to the superclass, provide a common + helper method and call the appropriate ctor in the fromXContent methods. + +commit 280a2f55d64b64663058e216273f8047dd01e425 +Author: Jim Ferenczi +Date: Thu Jun 7 13:37:29 2018 +0200 + + Fix random failure on SearchQueryIT#testTermExpansionExceptionOnSpanFailure + + This change moves an integration test that relies on setting + the value of a static variable (boolean max clause count) to + an unit test where we are sure that the same jvm is used to access + the static variable. + +commit 6ad721765604b9f69715acd7317944a0d0b088ca +Author: David Turner +Date: Thu Jun 7 12:38:57 2018 +0100 + + Remove reference to multiple fields with one name (#31127) + + If there is only one type per index then each field's name is unique. + +commit b5f05f676cfd0c6576f4ea19ab61729e0a38e6b5 +Author: Tanguy Leroux +Date: Thu Jun 7 10:48:31 2018 +0200 + + Remove BlobContainer.move() method (#31100) + + closes #30680 + +commit 749d39061a4a76b7264c443cb3235f72520ec0a3 +Author: Rafał Bigaj <4rafalbigaj@gmail.com> +Date: Thu Jun 7 10:43:31 2018 +0200 + + [Docs] Correct minor typos in templates.asciidoc (#31167) + +commit 458bca11bc4e1376f0c75171dd376be2bf64fae6 +Author: Adrien Grand +Date: Thu Jun 7 10:05:37 2018 +0200 + + Add a `feature_vector` field. (#31102) + + This field is similar to the `feature` field but is better suited to index + sparse feature vectors. A use-case for this field could be to record topics + associated with every documents alongside a metric that quantifies how well + the topic is connected to this document, and then boost queries based on the + topics that the logged user is interested in. + + Relates #27552 + +commit 75a676c70bb4333bde6f6b1cd89940c82d8299ad +Author: Nirmal Chidambaram +Date: Thu Jun 7 07:34:39 2018 +0000 + + Fail `span_multi` queries that exceeds boolean max clause limit (#30913) + + By default span_multi query will limit term expansions = boolean max clause. + This will limit high heap usage in case of high cardinality term + expansions. This applies only if top_terms_N is not used in inner multi + query. + +commit b30aa3137d1c56f032ec7c6cc4f3209c9e3172d5 +Author: Jim Ferenczi +Date: Thu Jun 7 09:29:26 2018 +0200 + + Reject long regex in query_string (#31136) + + This change applies the existing `index.max_regex_length` to regex queries + produced by the `query_string` query. + + Relates #28344 + +commit 5c6711b8a440a937dccb890e1e33cc75728bbc0c +Author: Simon Willnauer +Date: Thu Jun 7 07:39:28 2018 +0200 + + Use a `_recovery_source` if source is omitted or modified (#31106) + + Today if a user omits the `_source` entirely or modifies the source + on indexing we have no chance to re-create the document after it has + been added. This is an issue for CCR and recovery based on soft deletes + which we are going to make the default. This change adds an additional + recovery source if the source is disabled or modified that is only kept + around until the document leaves the retention policy window. + + This change adds a merge policy that efficiently removes this extra source + on merge for all document that are live and not in the retention policy window + anymore. + +commit 8be13615793a964560b96eea7681e70a8e62d5d2 +Author: Jason Tedor +Date: Wed Jun 6 22:51:43 2018 -0400 + + Adjust indentation in CLI scripts + + This commit adjusts the indentation in the CLI scripts to give a clear + visual indication that the line being indented is a continuation of the + previous line. + +commit bd3aabac970a47b7096c01195207e8c9acad3489 +Author: Tim Vernum +Date: Thu Jun 7 12:03:19 2018 +1000 + + [TEST] Make SSL restrictions update atomic (#31050) + + SSLTrustRestrictionsTests updates the restrictions YML file during the test run to change the set of restrictions. This update was small, but it wasn't atomic. + If the yml file is reloaded while empty or invalid, then it causes all SSL certificates to be considered invalid (until it is reloaded again), which could break the sniffing/administrative client that runs underneath the tests. + +commit 01b5a46c24d24895aa3a4eb78917534161fb76d4 +Author: Jason Tedor +Date: Wed Jun 6 21:57:58 2018 -0400 + + Pass main class by environment variable on Windows (#31156) + + A previous refactoring of the CLI scripts migrated all of the CLI tools + to shell to a common script, elasticsearch-cli. This approach is fine in + Bash where it is easy to tear arguments apart but it doesn't work so + well on Windows where quoting is insane. To avoid having to tear the + arguments apart to separate the first argument to elasticsearch-cli from + the remaining arguments, we instead choose a strategy where we can avoid + tearing the arguments apart. To do this, we will instead pass the main + class by an environment variable and then we can pass the arguments + straight through. This will let us avoid awful quoting issues on + Windows. This is the Windows side of that effort and the Bash side was + in a previous commit. + +commit 95795c8935b2c421ae47dcbb0e2300ac2905ec5a +Author: Jason Tedor +Date: Wed Jun 6 21:56:52 2018 -0400 + + Pass main class by environment variable (#31149) + + A previous refactoring of the CLI scripts migrated all of the CLI tools + to shell to a common script, elasticsearch-cli. This approach is fine in + Bash where it is easy to tear arguments apart but it doesn't work so + well on Windows where quoting is insane. To avoid having to tear the + arguments apart to separate the first argument to elasticsearch-cli from + the remaining arguments, we instead choose a strategy where we can avoid + tearing the arguments apart. To do this, we will instead pass the main + class by an environment variable and then we can pass the arguments + straight through. This will let us avoid awful quoting issues on + Windows. This is the non-Windows side of that effort and the Windows + side will be in a follow-up. + +commit 20a2f646e2df1a32e3263bbec595e11a48ceeb9a +Author: Jason Tedor +Date: Wed Jun 6 19:53:49 2018 -0400 + + Fix off-by-one error in chunks coordinator (#31147) + + This commit fixes an off-by-error in the chunks coordinator where the + batches would be of size one more than the batch size. + +commit 7f0c2e89c2f8a428978b2b873ff0c33834c0f455 +Author: Lisa Cawley +Date: Wed Jun 6 14:46:20 2018 -0700 + + [DOCS] Moves X-Pack setup to docs (#31145) + +commit 4158387554ff2ff18fd0fbb10f8e6f2de55d35c8 +Author: Tim Brooks +Date: Wed Jun 6 15:36:13 2018 -0600 + + Cleanup nio http thread names (#31148) + + This is related to #28898. This commit adds the acceptor thread name to + the method checking if this thread is a transport thread. Additionally, + it modifies the nio http transport to use the same worker name as the + netty4 http server transport. + +commit be4a101ea1824d4e6be1dec7341ddbe1d620d1c5 +Author: Luca Cavanna +Date: Wed Jun 6 23:17:45 2018 +0200 + + Add high-level client methods that accept RequestOptions (#31069) + + With #30490 we have introduced a new way to provide request options + whenever sending a request using the high-level REST client. Before you + could provide headers as the last argument varargs of each API method, + now you can provide `RequestOptions` that in the future will allow to + provide more options which can be specified per request. + + This commit deprecates all of the client methods that accept a `Header` + varargs argument in favour of new methods that accept `RequestOptions` + instead. For some API we don't even go through deprecation given that + they were not released since they were added, hence in that case we can + just move them to the new method. + +commit 68827fc0465abe1e365815fd344e900f593e5008 +Author: Lisa Cawley +Date: Wed Jun 6 13:25:36 2018 -0700 + + [DOCS] Enables testing for monitoring examples (#31119) + +commit b4514d3cc12c18cea5395c3dcb7f41455ec4ec6b +Author: Lisa Cawley +Date: Wed Jun 6 12:39:24 2018 -0700 + + [DOCS] Moves ML node info to docs (#31142) + +commit 67e73b4df481f3902a4aeee78d34532297ff7b6b +Author: Tim Brooks +Date: Wed Jun 6 11:59:54 2018 -0600 + + Combine accepting selector and socket selector (#31115) + + This is related to #27260. This commit combines the AcceptingSelector + and SocketSelector classes into a single NioSelector. This change + allows the same selector to handle both server and socket channels. This + is valuable as we do not necessarily want a dedicated thread running for + accepting channels. + + With this change, this commit removes the configuration for dedicated + accepting selectors for the normal transport class. The accepting + workload for new node connections is likely low, meaning that there is + no need to dedicate a thread to this process. + +commit dc4bb62a7893f15cd5d48eb710bf95359a3c6177 +Author: Nik Everett +Date: Wed Jun 6 13:51:54 2018 -0400 + + QA: Remove mistaken timeout + + I pushed a test that `assertBusy`s for a whole hour accidentally. I was + testing something and forgot to revert my local hack but caught it on + backport. This removes it. + +commit 6b6882900d5e36aa15ac784ee4ad61797e951de9 +Merge: 6b0008c6686 45537c59e59 +Author: Tal Levy +Date: Wed Jun 6 10:08:03 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 45537c59e59287216099138f14cf58c5416503d0 +Author: Lisa Cawley +Date: Wed Jun 6 10:05:32 2018 -0700 + + [DOCS] Moves X-Pack settings to docs folder (#31120) + +commit 7c59e7690e04b87c148d9b9cb1c0000f533a4da7 +Author: Nik Everett +Date: Wed Jun 6 11:59:16 2018 -0400 + + QA: Switch xpack rolling upgrades to three nodes (#31112) + + This is much more realistic and can find more issues. This causes the + "mixed cluster" tests to be run twice so I had to fix the tests to work + in that case. In most cases I did as little as possible to get them + working but in a few cases I went a little beyond that to make them + easier for me to debug while getting them to work. My test changes: + + 1. Remove the "basic indexing" tests and replace them with a copy of the + tests used in the OSS. We have no way of sharing code between these two + projects so for now I copy. + 2. Skip the a few tests in the "one third" upgraded scenario: + * creating a scroll to be reused when the cluster is fully upgraded + * creating some ml data to be used when the cluster is fully ugpraded + 3. Drop many "assert yellow and that the cluster has two nodes" + assertions. These assertions duplicate those made by the wait condition + and they fail now that we have three nodes. + 4. Switch many "assert green and that the cluster has two nodes" to 3 + nodes. These assertions are unique from the wait condition and, while + I imagine they aren't required in all cases, now is not the time to + find that out. Thus, I made them work. + 5. Rework the index audit trail test so it is more obvious that it is + the same test expecting different numbers based on the shape of the + cluster. The conditions for which number are expected are fairly + complex because the index audit trail is shut down until the template + for it is upgraded and the template is upgraded when a master node is + elected that has the new version of the software. + 6. Add some more information to debug the index audit trail test because + it helped me figure out what was going on. + + I also dropped the `waitCondition` from the `rolling-upgrade-basic` + tests because it wasn't needed. + + Closes #25336 + +commit bf1152fcc62b69f67b4ead4f104b632e4ac3ce39 +Author: Jason Tedor +Date: Wed Jun 6 11:03:57 2018 -0400 + + Use follower primary term when applying operations (#31113) + + The primary shard copy on the following has authority of the replication + operations that occur on the following side in cross-cluster + replication. Yet today we are using the primary term directly from the + operations on the leader side. Instead we should be replacing the + primary term on the following side with the primary term of the primary + on the following side. This commit does this by copying the translog + operations with the corrected primary term. This ensures that we use + this primary term while applying the operations on the primary, and when + replicating them across to the replica (where the replica request was + carrying the primary term of the primary shard copy on the follower). + +commit 6fd4eb52b8261581cf3b7e5c3178fe7940d1b00f +Author: Lisa Cawley +Date: Wed Jun 6 07:49:15 2018 -0700 + + [DOCS] Moves commands to docs folder (#31114) + +commit e9fe371e41bcb6f8bc7b90b9cbbd2c74e0145a79 +Author: Adrien Grand +Date: Wed Jun 6 16:46:11 2018 +0200 + + Give the engine the whole index buffer size on init. (#31105) + + Currently the engine is initialized with a hardcoded 256MB of RAM. Elasticsearch + may never use more than that for a given shard, `IndexingMemoryController` only + has the power to flush segments to disk earlier in case multiple shards are + actively indexing and use too much memory. + + While this amount of memory is enough for an index with few fields and larger + RAM buffers are not expected to improve indexing speed, this might actually be + little for an index that has many fields. + + Kudos to @bleskes for finding it out when looking into a user who was reporting + a **much** slower indexing speed when upgrading from 2.x to 5.6 with an index + that has about 20,000 fields. + +commit 1dca00deb93bdc104f6ebb8814bf43a42b9273d9 +Author: Yannick Welsch +Date: Wed Jun 6 16:38:37 2018 +0200 + + Remove extra checks from HdfsBlobContainer (#31126) + + This commit saves one network roundtrip when reading or deleting files from an HDFS repository. + +commit 515a23360d4791499e7675c38bbad9e6e244fc7b +Author: Yannick Welsch +Date: Wed Jun 6 16:38:06 2018 +0200 + + Do not check for S3 blob to exist before writing (#31128) + + In #19749 an extra check was added before writing each blob to ensure that we would not be + overriding an existing blob. Due to S3's weak consistency model, this check was best effort. To + make matters worse, however, this resulted in a HEAD request to be done before every PUT, in + particular also when PUTTING a new object. The approach taken in #19749 worsened our + consistency guarantees for follow-up snapshot actions, as it made it less likely for new files that + had been written to be available for reads. + + This commit therefore removes this extra check. Due to the weak consistency model, this check + was a best effort thing anyway, and there's currently no way to prevent accidental overrides on S3. + +commit 8aa58887e2a15ebe49358b0a33f1f4ec81dd5836 +Author: Jay Modi +Date: Wed Jun 6 08:18:56 2018 -0600 + + Security: make native realm usage stats accurate (#30824) + + The native realm's usage stats were previously pulled from the cache, + which only contains the number of users that had authenticated in the + past 20 minutes. This commit changes this so that we pull the current + value from the security index by executing a search request. In order + to support this, the usage stats for realms is now asynchronous so that + we do not block while waiting on the search to complete. + +commit f4a412fe21b2ae595306e6bd261f5fc4d6374388 +Author: Luca Cavanna +Date: Wed Jun 6 16:13:02 2018 +0200 + + Remove RestGetAllMappingsAction (#31129) + + We currently have a specific REST action to retrieve all indices and types mappings, which used internally the get index API. This doesn't seem to be required anymore though as the existing RestGetMappingAction could as well take the requests with no indices and types specified. + + This commit removes the RestGetAllMappingsAction in favour of using RestGetMappingAction also for requests that don't specify indices nor types. + +commit a9af5ca6387cc0fa6d4e1425f57034d94713a328 +Author: Yannick Welsch +Date: Wed Jun 6 14:32:37 2018 +0200 + + [TEST] Reenable UnicastZenPingTests.testSimplePings + +commit 3767bdc98d974282e4b759fe9a072b0d188fe4f2 +Author: David Kyle +Date: Wed Jun 6 13:21:16 2018 +0100 + + [ML][DOCS] Add example of top N derivative aggregation (#31109) + + Add example of top N derivative aggregation to the ML datafeed docs + +commit 0c8c6191816e464c065672d1b4b9cd91b26bb0e5 +Author: Albert Zaharovits +Date: Wed Jun 6 14:59:04 2018 +0300 + + Fix audit index template upgrade loop (#30779) + + The Index Audit trail allows the override of the template index + settings with settings specified on the conf file. + A bug will manifest when such conf file settings are specified + for templates that need to be upgraded. The bug is an endless + upgrade loop because the upgrade, although successful, is + not reckoned as such by the upgrade service. + +commit 23d156f02306c0373c0689c0004a53a9e7c1e0ee +Author: Luca Cavanna +Date: Wed Jun 6 13:39:02 2018 +0200 + + Move RestGetSettingsAction to RestToXContentListener (#31101) + +commit d230548401cb9e44072ad9c64480b74b82cfadbc +Author: Jason Tedor +Date: Wed Jun 6 05:09:55 2018 -0400 + + Remove use of deprecated methods to perform request (#31117) + + The old perform request methods on the REST client have been deprecated + in favor using request-flavored methods. This commit addresses the use + of these deprecated methods in the CCR test suite. + +commit d09d60858a6acdaa794201d0fca0115a4d97cb5d +Author: Colin Goodheart-Smithe +Date: Wed Jun 6 09:32:45 2018 +0100 + + [DOCS] Clarify nested datatype introduction (#31055) + +commit 0c9d4cb4178f55b88f0da1dfc41939405c3a8d00 +Author: Christoph Büscher +Date: Wed Jun 6 09:58:16 2018 +0200 + + Fix expectation on parsing exception (#31108) + + The structure of the expected exception slightly changed, the change adapts the + assertions accordingly. + + Closes #31104 + +commit 1cee45e768b1b2d37961d5504563971b3bd0991d +Author: Christoph Büscher +Date: Wed Jun 6 09:53:14 2018 +0200 + + [Docs] Delete superfluous callouts (#31111) + + Those callout create rendering problems on the subsequent page. + + Closes #30532 + +commit e9bd92fc764894445452b362b428aca2358288aa +Author: Martijn van Groningen +Date: Wed Jun 6 07:44:01 2018 +0200 + + fixed typo + +commit 735d0e671aa71a9c6ecf111eeca4d0355ea8167f +Author: Martijn van Groningen +Date: Wed Jun 6 07:40:21 2018 +0200 + + Make PreBuiltAnalyzerProviderFactory plugable via AnalysisPlugin and + + move `finger_print`, `pattern` and `standard_html_strip` analyzers + to analysis-common module. (both AnalysisProvider and PreBuiltAnalyzerProvider) + + Changed PreBuiltAnalyzerProviderFactory to extend from PreConfiguredAnalysisComponent and + changed to make sure that predefined analyzers are always instantiated with the current + ES version and if an instance is requested for a different version then delegate to PreBuiltCache. + This is similar to the behaviour that exists today in AnalysisRegistry.PreBuiltAnalysis and + PreBuiltAnalyzerProviderFactory. (#31095) + + Relates to #23658 + +commit 805648848d50b89e4c85d2f35a8de7456c67a694 +Author: Jason Tedor +Date: Tue Jun 5 19:56:22 2018 -0400 + + Add check for feature aware implementations (#31081) + + This commit adds a check that any class in X-Pack that is a feature + aware custom also implements the appropriate mix-in interface in + X-Pack. These interfaces provide a default implementation of + FeatureAware#getRequiredFeature that returns that x-pack is the required + feature. By implementing this interface, this gives a consistent way for + X-Pack feature aware customs to return the appopriate required feature + and this check enforces that all such feature aware customs return the + appropriate required feature. + +commit 6e109e90c3873cfe815764e161b9869817b88c22 +Merge: 755a25a41e5 7c05f69c390 +Author: Jason Tedor +Date: Tue Jun 5 19:53:37 2018 -0400 + + Merge remote-tracking branch 'elastic/master' into ccr + + * elastic/master: + [DOCS] Creates rest-api folder in docs + [Rollup] Disallow index patterns that match the rollup index (#30491) + Add cors support to NioHttpServerTransport (#30827) + [DOCS] Fixes security example (#31082) + Allow terms query in _rollup_search (#30973) + +commit 7c05f69c390172f789e7bd7f13af19175cfb36fb +Author: lcawl +Date: Tue Jun 5 16:43:55 2018 -0700 + + [DOCS] Creates rest-api folder in docs + +commit a1c9def64ecd09afc80c181b65150375ad8fa23b +Author: Zachary Tong +Date: Tue Jun 5 15:00:34 2018 -0400 + + [Rollup] Disallow index patterns that match the rollup index (#30491) + + We should not allow the user to configure index patterns that also match + the index which stores the rollup index. + + For example, it is quite natural for a user to specify `metricbeat-*` + as the index pattern, and then store the rollups in `metricbeat-rolled`. + This will start throwing errors as soon as the rollup index is created + because the indexer will try to search it. + + Note: this does not prevent the user from matching against existing + rollup indices. That should be prevented by the field-level validation + during job creation. + +commit 755a25a41e508bc9e88a0572f5282307323b1077 +Author: Nhat Nguyen +Date: Tue Jun 5 14:11:19 2018 -0400 + + Replace exact numDocs by soft-del count in SegmentInfo (#31086) + + This PR adapts/utilizes recent enhancements in Lucene-7.4: + + - Replaces exactNumDocs by the soft-deletes count in SegmentCommitInfo. + This enhancement allows us to back out changes introduced in #30228. + + - Always configure the soft-deletes field in IWC + +commit 91de0ba378ad6b6a22d6baacb9c7d7a8757221d3 +Author: Nhat Nguyen +Date: Mon Jun 4 14:18:46 2018 -0400 + + Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (#31073) + + This snapshot includes: + - LUCENE-8341: Record soft deletes in SegmentCommitInfo which will resolve #30851 + - LUCENE-8335: Enforce soft-deletes field up-front + +commit 05ee0f8b6e79f900d3d627adcc2f52182a1d472a +Author: Tim Brooks +Date: Tue Jun 5 10:09:20 2018 -0600 + + Add cors support to NioHttpServerTransport (#30827) + + This is related to #28898. This commit adds cors support to the nio http + transport. Most of the work is copied directly from the netty module + implementation. Additionally, this commit adds tests for the nio http + channel. + +commit 840a3bd5a6979d35255b85b38418ceda462ccc52 +Author: Lisa Cawley +Date: Tue Jun 5 08:50:06 2018 -0700 + + [DOCS] Fixes security example (#31082) + +commit 6b0008c6686c700c9a9e7afbaf1e5edb8b2fa282 +Merge: 2820dba2d7c 7f850bb8ced +Author: Tal Levy +Date: Tue Jun 5 08:09:02 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 2b5c0d77e984587b01f52f495aff214a683a53fe +Merge: 530089f132b 14c40885bec +Author: Nhat Nguyen +Date: Tue Jun 5 10:58:35 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Removing erroneous repeat + Adapt bwc versions after backporting #30983 to 6.4 + [Tests] Muting RatedRequestsTests#testXContentParsingIsNotLenient + TEST: Retry synced-flush if ongoing ops on primary (#30978) + Fix docs build. + Only auto-update license signature if all nodes ready (#30859) + Add BlobContainer.writeBlobAtomic() (#30902) + Add a doc value format to binary fields. (#30860) + +commit 7f850bb8ced66c6290a41dfd2b4728ae704b656a +Author: Jim Ferenczi +Date: Tue Jun 5 16:51:14 2018 +0200 + + Allow terms query in _rollup_search (#30973) + + This change adds the `terms` query to the list of accepted queries + for the _rollup_search endpoint. + +commit 14c40885beca1ffd7def507b2ede5284a6f1564b +Author: Christoph Büscher +Date: Tue Jun 5 16:34:19 2018 +0200 + + Removing erroneous repeat + +commit 81172c0c31e99cbad073471a0595d63b80d6dbe4 +Author: Christoph Büscher +Date: Tue Jun 5 15:47:53 2018 +0200 + + Adapt bwc versions after backporting #30983 to 6.4 + +commit 4624ba5e100e19bb624e0b72e6b55eb171d6fef4 +Author: Christoph Büscher +Date: Tue Jun 5 15:29:00 2018 +0200 + + [Tests] Muting RatedRequestsTests#testXContentParsingIsNotLenient + +commit 4b893c190068d6cf83a8f341c3f9ca535fb5738e +Author: Nhat Nguyen +Date: Tue Jun 5 09:02:13 2018 -0400 + + TEST: Retry synced-flush if ongoing ops on primary (#30978) + + When the last indexing operation is completed, we will fire a global + checkpoint sync. Since a global checkpoint sync request is a replication + request, it will acquire an index shard permit on the primary when + executing. If this happens at the same time while we are issuing the + synced-flush, the synced-flush request will fail as it thinks there are + in-flight operations. We can avoid such situation by retrying another + synced-flush if the current request fails due to ongoing operations on + the primary. + + Closes #29392 + +commit 1af6d20efe83722515a0c449cdb2e8e7288686f5 +Author: Adrien Grand +Date: Tue Jun 5 14:54:24 2018 +0200 + + Fix docs build. + +commit 3b98c26d03799239178752a8adf12c5b8386a5b8 +Author: Yannick Welsch +Date: Tue Jun 5 13:43:04 2018 +0200 + + Only auto-update license signature if all nodes ready (#30859) + + Allows rolling restart from 6.3 to 6.4. + + Relates to #30731 and #30251 + +commit 9531b7bbcbbd3391d9f605c2c4aa42a123aaf3bd +Author: Tanguy Leroux +Date: Tue Jun 5 13:00:43 2018 +0200 + + Add BlobContainer.writeBlobAtomic() (#30902) + + This commit adds a new writeBlobAtomic() method to the BlobContainer + interface that can be implemented by repository implementations which + support atomic writes operations. + + When the BlobContainer implementation does not provide a specific + implementation of writeBlobAtomic(), then the writeBlob() method is used. + + Related to #30680 + +commit f2892f1bed9aedc7e62b05a01f7679e116f6b634 +Author: Adrien Grand +Date: Tue Jun 5 12:42:13 2018 +0200 + + Add a doc value format to binary fields. (#30860) + + This will be necessary for the `docvalue_fields` option to work correctly once + we use the field's doc-value format to format doc-value fields. Binary values + are formatted as base64-encoded strings. + +commit 530089f132bc7914dc52820832c37ca1813cbd16 +Merge: a76dcafe760 0fad7cc99a2 +Author: Martijn van Groningen +Date: Tue Jun 5 10:37:38 2018 +0200 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: + Take into account the return value of TcpTransport.readMessageLength(...) in Netty4SizeHeaderFrameDecoder + Move caching of the size of a directory to `StoreDirectory`. (#30581) + Clarify docs about boolean operator precedence. (#30808) + Docs: remove notes on sparsity. (#30905) + Fix MatchPhrasePrefixQueryBuilderTests#testPhraseOnFieldWithNoTerms + run overflow forecast a 2nd time as regression test for elastic/ml-cpp#110 (#30969) + Improve documentation of dynamic mappings. (#30952) + Decouple MultiValueMode. (#31075) + Docs: Clarify constraints on scripted similarities. (#31076) + Update get.asciidoc (#31084) + +commit 0fad7cc99a29e1dd16a1f1cf6f82965a97396ca5 +Author: Martijn van Groningen +Date: Tue Jun 5 10:35:47 2018 +0200 + + Take into account the return value of TcpTransport.readMessageLength(...) in Netty4SizeHeaderFrameDecoder + + (#31057) + +commit 03dcf22e063ad5a60620ec19bdd575f13dab1f34 +Author: Adrien Grand +Date: Tue Jun 5 09:01:12 2018 +0200 + + Move caching of the size of a directory to `StoreDirectory`. (#30581) + + In spite of the existing caching, I have seen a number of nodes hot threads + where one thread had been spending all its cpu on computing the size of a + directory. I am proposing to move the computation of the size of the directory + to `StoreDirectory` in order to skip recomputing the size of the directory if + no changes have been made. This should help with users that have read-only + indices, which is very common for time-based indices. + +commit 984523dda9dd71ec0b9609568d7b6dc7d64a9047 +Author: Adrien Grand +Date: Tue Jun 5 08:59:17 2018 +0200 + + Clarify docs about boolean operator precedence. (#30808) + + Unfortunately, the classic queryparser does not honor the usual precedence + rules of boolean operators. See + https://issues.apache.org/jira/browse/LUCENE-3674. + +commit 21fe6159d49340dcfad0b9273376815012c9d20f +Author: Adrien Grand +Date: Tue Jun 5 08:58:52 2018 +0200 + + Docs: remove notes on sparsity. (#30905) + + Sparsity is less of a concern since 6.0. + + Closes #30833 + +commit 990442fdb7116823474d089b60e33cc36ddf217b +Author: Jim Ferenczi +Date: Tue Jun 5 08:52:29 2018 +0200 + + Fix MatchPhrasePrefixQueryBuilderTests#testPhraseOnFieldWithNoTerms + + The test cannot run when no type is registered. + + Relates #30450 + +commit 5e48ba7cbdf843e53a3c2a55333d6dd6c0586267 +Author: Hendrik Muhs +Date: Tue Jun 5 08:52:06 2018 +0200 + + run overflow forecast a 2nd time as regression test for elastic/ml-cpp#110 (#30969) + + Improve test to run overflow forecast a 2nd time as regression test for elastic/ml-cpp#110 + +commit 500094f5c8d7579ae3530b16e15b9af36d987584 +Author: Adrien Grand +Date: Tue Jun 5 08:51:52 2018 +0200 + + Improve documentation of dynamic mappings. (#30952) + + Closes #30939 + +commit cc552350308c34bb721485dd9aac6b933c7d7a02 +Author: Adrien Grand +Date: Tue Jun 5 08:51:20 2018 +0200 + + Decouple MultiValueMode. (#31075) + + Currently this class takes care of moth selecting the relevant value, and + replacing missing values if any. This is fine for sorting, which always needs + to do both at the same time, but we also have a number of aggregations and + script utils that need to retain information about missing values so this change + proposes to decouple selection of the relevant value and replacement of missing + values. + +commit f5073813ef2636268aaf5f828efb08c40171e070 +Author: Adrien Grand +Date: Tue Jun 5 08:51:00 2018 +0200 + + Docs: Clarify constraints on scripted similarities. (#31076) + + Scripted similarities provide a lot of flexibility but they still need to obey + some rules to not confuse Lucene. + +commit 2820dba2d7cbbd39e32df7216bbb00a15a0784ad +Merge: c3e9a6040f7 c7c0acc2c73 +Author: Tal Levy +Date: Mon Jun 4 19:59:49 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit c3e9a6040f7427b5d04c14671230033c3120f269 +Author: Tal Levy +Date: Mon Jun 4 16:10:01 2018 -0700 + + update retry-action to be in line with indices requests (#31042) + +commit c7c0acc2c73fe5b6244c2ab265d9c6b9532ce9b7 +Author: Dmitri Moore +Date: Mon Jun 4 15:40:18 2018 -0700 + + Update get.asciidoc (#31084) + + Fix typo + + (cherry picked from commit 51fbf8a3d3486fcf8a66dd6a6fe0940482b0f2d8) + +commit a76dcafe760e7f6b08bbbc3495bc7069f8fe63f4 +Author: Nhat Nguyen +Date: Mon Jun 4 17:46:46 2018 -0400 + + Revert "Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (#31073)" + + This reverts commit abe61159a82ad9277788d3de0796f0986bb11e55. + +commit 646e1c01b0d3fe10dc3e7ee2b1b250982c999623 +Merge: 6ee6404e94b b22a055bcff +Author: Nhat Nguyen +Date: Mon Jun 4 16:32:43 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add get mappings support to high-level rest client (#30889) + Fix index prefixes to work with span_multi (#31066) + [DOCS] Removes redundant authorization pages + [DOCS] Re-adds custom realm + Change ObjectParser exception (#31030) + Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (#31073) + +commit b22a055bcffa3a3667b3aae6a3420442e889ad1c +Author: Lee Hinman +Date: Mon Jun 4 14:31:08 2018 -0600 + + Add get mappings support to high-level rest client (#30889) + + This adds support for the get mappings API to the high level rest client. + + Relates to #27205 + +commit f94a75778c3ab833a2ce15eec7a44f187aa9ccee +Author: Jim Ferenczi +Date: Mon Jun 4 21:48:56 2018 +0200 + + Fix index prefixes to work with span_multi (#31066) + + * Fix index prefixes to work with span_multi + + Text fields that use `index_prefixes` can rewrite `prefix` queries into + `term` queries internally. This commit fix the handling of this rewriting + in the `span_multi` query. + This change also copies the index options of the text field into the + prefix field in order to be able to run positional queries. This is mandatory + for `span_multi` to work but this could also be useful to optimize `match_phrase_prefix` + queries in a follow up. Note that this change can only be done on indices created + after 6.3 since we set the index options to doc only in this version. + + Fixes #31056 + +commit 32902b78dbe54658f3cb087779afceec549f9cad +Author: Tal Levy +Date: Mon Jun 4 12:06:10 2018 -0700 + + fix merge conflicts with latest master + +commit 024400bcb8b2de5b0b5c3648865eeded73df87ac +Author: lcawl +Date: Mon Jun 4 12:02:50 2018 -0700 + + [DOCS] Removes redundant authorization pages + +commit 6ee6404e94b5b30f0e12321a2f59e179967a4a5d +Author: Nhat Nguyen +Date: Mon Jun 4 14:48:04 2018 -0400 + + Adapt changes in PersistentTaskParams + + Relates #31045 + +commit 87abb49145b73dc5f183006918a09a408688e0f7 +Author: Nhat Nguyen +Date: Mon Jun 4 14:47:22 2018 -0400 + + Adapt changes in AcknowledgeResponse + + Relates #30983 + +commit d9a0a02e89134ebaef323ad6a02b1d7c541842d6 +Merge: 5fd759b6936 852df128a55 +Author: Nhat Nguyen +Date: Mon Jun 4 14:32:28 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Match phrase queries against non-indexed fields should throw an exception (#31060) + In the internal highlighter APIs, use the field type as opposed to the mapper. (#31039) + [DOCS] Removes duplicated authentication pages + Enable customizing REST tests blacklist (#31074) + Make sure KeywordFieldMapper#clone preserves split_queries_on_whitespace. (#31049) + [DOCS] Moves machine learning overview to stack-docs + [ML] Add secondary sort to ML events (#31063) + [Rollup] Specialize validation exception for easier management (#30339) + Adapt bwc versions after backporting #31045 to 6.3 + Remove usage of explicit type in docs (#29667) + Share common readFrom/writeTo code in AcknowledgeResponse (#30983) + Adapt bwc versions after backporting #31045 to 6.x + Mute MatchPhrase*QueryBuilderTests + [Docs] Fix typo in watcher conditions documentation (#30989) + Remove wrong link in index phrases doc + Move pipeline APIs to ingest namespace (#31027) + [DOCS] Fixes accounting setting names (#30863) + [DOCS] Rewords _field_names documentation (#31029) + Index phrases (#30450) + Remove leftover debugging from PTCMDT + Fix PTCMDT#testMinVersionSerialization + Make Persistent Tasks implementations version and feature aware (#31045) + +commit 409da09a81e569e1d4be993d5045f85364a0df89 +Author: lcawl +Date: Mon Jun 4 11:30:11 2018 -0700 + + [DOCS] Re-adds custom realm + +commit 3f87c7950019316f59edd0199852bd3233db76b2 +Author: Christoph Büscher +Date: Mon Jun 4 20:20:37 2018 +0200 + + Change ObjectParser exception (#31030) + + ObjectParser should throw XContentParseExceptions, not IAE. A dedicated parsing + exception can includes the place where the error occurred. + + Closes #30605 + +commit abe61159a82ad9277788d3de0796f0986bb11e55 +Author: Nhat Nguyen +Date: Mon Jun 4 14:18:46 2018 -0400 + + Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (#31073) + + This snapshot includes: + - LUCENE-8341: Record soft deletes in SegmentCommitInfo which will resolve #30851 + - LUCENE-8335: Enforce soft-deletes field up-front + +commit 852df128a5544dbafaf7f901e31f3d88431c767e +Author: Alan Woodward +Date: Mon Jun 4 19:12:45 2018 +0100 + + Match phrase queries against non-indexed fields should throw an exception (#31060) + + When `lenient=false`, attempts to create match phrase queries with custom analyzers against non-text fields will throw an IllegalArgumentException. + + Also changes `*Match*QueryBuilderTests` so that it avoids this scenario + + Fixes #31061 + +commit 609de08126362497b17f25472665941f413faad7 +Author: Julie Tibshirani +Date: Mon Jun 4 11:12:03 2018 -0700 + + In the internal highlighter APIs, use the field type as opposed to the mapper. (#31039) + +commit 53357e7d141df23da27a30c1b50fda70533370f4 +Author: lcawl +Date: Mon Jun 4 10:58:41 2018 -0700 + + [DOCS] Removes duplicated authentication pages + +commit be55da18c21531b2d0c07a5e0c5c54893e6dce29 +Author: Jason Tedor +Date: Mon Jun 4 13:35:49 2018 -0400 + + Enable customizing REST tests blacklist (#31074) + + This commit enables adding additional REST tests to the blacklist for + builds that already define tests.rest.blacklist. + +commit 2294c3bea43e7e808c2c10f32fd659b226b4f89a +Merge: e2420525209 30a8f9d9480 +Author: Tal Levy +Date: Mon Jun 4 09:26:53 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 30a8f9d948050a2ca0fb93499471d8aee782c774 +Author: Julie Tibshirani +Date: Mon Jun 4 08:42:32 2018 -0700 + + Make sure KeywordFieldMapper#clone preserves split_queries_on_whitespace. (#31049) + +commit 7e565797e7245ff7af48907ed603f2f02fe92d01 +Author: lcawl +Date: Thu May 31 11:44:30 2018 -0700 + + [DOCS] Moves machine learning overview to stack-docs + +commit 16d1f05045c79b323e26b82d6b61af3fc539d93c +Author: David Kyle +Date: Mon Jun 4 16:31:35 2018 +0100 + + [ML] Add secondary sort to ML events (#31063) + +commit b422978c0ca6e28416d13212c4cfccfbe6b57dc5 +Author: Zachary Tong +Date: Mon Jun 4 10:28:36 2018 -0400 + + [Rollup] Specialize validation exception for easier management (#30339) + + Extends ActionRequestValidationException with a rollup-specific version + to make it easier to handle mapping validation issues on the client + side. + + The type will now be `rollup_action_request_validation_exception` + instead of `action_request_validation_exception` + +commit 167b9b3656c7a5bf33a971a32f9548ce83f8cf0b +Author: Boaz Leskes +Date: Sun Jun 3 21:55:34 2018 +0200 + + Adapt bwc versions after backporting #31045 to 6.3 + +commit 526f90692747ff29c07ee4a3a6f177160b5a8fbc +Author: Itamar Syn-Hershko +Date: Mon Jun 4 16:42:29 2018 +0300 + + Remove usage of explicit type in docs (#29667) + +commit 11b11f6f4cbae65a76834d45b2697906ead4e003 +Author: Christoph Büscher +Date: Mon Jun 4 15:10:02 2018 +0200 + + Share common readFrom/writeTo code in AcknowledgeResponse (#30983) + + The majority of Responses inheriting from AcknowledgeResponse implement + the readFrom and writeTo serialization method in the same way. Moving this + as a default into AcknowledgeResponse and letting the few exceptions that + need a slightly different implementation handle this themselves saves a lot + of duplication. + +commit ccb78c2fdf92d0d0eb09640d8e3e3491c6d474fe +Author: Boaz Leskes +Date: Sun Jun 3 21:55:34 2018 +0200 + + Adapt bwc versions after backporting #31045 to 6.x + +commit 146965f3ecf0bab9343a62fca73efb233bd0b0f4 +Author: Daniel Mitterdorfer +Date: Mon Jun 4 14:03:01 2018 +0200 + + Mute MatchPhrase*QueryBuilderTests + + Relates #31061 + +commit b665d591f1f58ae18eacc37b305aebd7edf89608 +Author: Michael Russell +Date: Mon Jun 4 13:00:57 2018 +0200 + + [Docs] Fix typo in watcher conditions documentation (#30989) + +commit fa6b7266eb4d9440137005fde3d719855e43d70c +Author: Jim Ferenczi +Date: Mon Jun 4 12:13:55 2018 +0200 + + Remove wrong link in index phrases doc + + Relates #30450 + +commit e24205252094fac076b6332ee753a1f3b0d01afd +Merge: 1c40da8fbfd 3809450bf3a +Author: Colin Goodheart-Smithe +Date: Mon Jun 4 10:39:53 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 1c40da8fbfdd900332b1ae7d202503f5f3e704a8 +Merge: 62da5dc17b0 360b09f1485 +Author: Colin Goodheart-Smithe +Date: Mon Jun 4 10:14:17 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 3809450bf3aaa48317b6e717525550b818315847 +Author: Sohaib Iftikhar +Date: Mon Jun 4 10:34:55 2018 +0200 + + Move pipeline APIs to ingest namespace (#31027) + +commit 360b09f14858d8062d79053115c6366eafd6af8e +Author: Colin Goodheart-Smithe +Date: Mon Jun 4 09:19:38 2018 +0100 + + [DOCS] Fixes accounting setting names (#30863) + + The documentation for the account circuit breaker listed the settings for it's limit and overhead to be `network.breaker.accounting.limit` and `network.breaker.accounting.overhead` when in `HieratchyCircuitBreakerService` it seems the settings are actually `indices.breaker.accounting.limit` and `indices.breaker.accounting.overhead`. + +commit 1efb1aae28f7be63212e507a57f7c2a870e5815b +Author: Colin Goodheart-Smithe +Date: Mon Jun 4 09:17:11 2018 +0100 + + [DOCS] Rewords _field_names documentation (#31029) + + * [DOCS] Rewords _field_names documentation + + Corrects the language around when we write to `_field_names` and when you might want to disable it given that n recent versions it does not carry the indexing overhead it once did. + + Relates to #30862 + + * Update wording following review + +commit 0427339ab04912d51c8bb53cc3a9dbda317f3dd2 +Author: Alan Woodward +Date: Mon Jun 4 08:50:35 2018 +0100 + + Index phrases (#30450) + + Specifying `index_phrases: true` on a text field mapping will add a subsidiary + [field]._index_phrase field, indexing two-term shingles from the parent field. + The parent analysis chain is re-used, wrapped with a FixedShingleFilter. + + At query time, if a phrase match query is executed, the mapping will redirect it + to run against the subsidiary field. + + This should trade faster phrase querying for a larger index and longer indexing + times. + + Relates to #27049 + +commit dc8a4fb4606acb7a7ac336803920a21fc4b8e43d +Author: Jason Tedor +Date: Sun Jun 3 21:53:21 2018 -0400 + + Remove leftover debugging from PTCMDT + + This commit removes some leftover debugging statements. + +commit 5667b08aaa49df5a24c5839e7889f0365caf19e8 +Author: Jason Tedor +Date: Sun Jun 3 21:35:01 2018 -0400 + + Fix PTCMDT#testMinVersionSerialization + + This commit fixes an issue with + PersistentTasksCustomMetaDataTests#testMinVersionSerialization. There + were two problems here: + - some versions do not have future compatible version (e.g., betas) + - the feature logic was incorrect + +commit a7ceefe93fd36574eb2c661c2e7079a0ceb5e0b0 +Author: Boaz Leskes +Date: Sun Jun 3 21:51:08 2018 +0200 + + Make Persistent Tasks implementations version and feature aware (#31045) + + With #31020 we introduced the ability for transport clients to indicate what features they support + in order to make sure we don't serialize object to them they don't support. This PR adapts the + serialization logic of persistent tasks to be aware of those features and not serialize tasks that + aren't supported. + + Also, a version check is added for the future where we may add new tasks implementations and + need to be able to indicate they shouldn't be serialized both to nodes and clients. + + As the implementation relies on the interface of `PersistentTaskParams`, these are no longer + optional. That's acceptable as all current implementation have them and we plan to make + `PersistentTaskParams` more central in the future. + + Relates to #30731 + +commit 5fd759b69367262d66018d8965b43ed11eab5b1f +Merge: 9564b601940 65d3f0efcaf +Author: Nhat Nguyen +Date: Sat Jun 2 08:23:19 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Adapt transport tests for the extra byte introduced in #31020 + +commit 65d3f0efcaf2052f85e07e742f7d2980fe26d593 +Author: Boaz Leskes +Date: Sat Jun 2 13:04:40 2018 +0200 + + Adapt transport tests for the extra byte introduced in #31020 + + We now serialize a feature array, which takes an extra byte when empty. + +commit 9564b6019408ac2544a183ff0d13ebe4f25c2f63 +Author: Nhat Nguyen +Date: Fri Jun 1 23:09:59 2018 -0400 + + Adjust CCR Actions after RequestBuilder is removed + + CCR side of #30966 + +commit a527095b8d84d5eaeb47b4fd82dd097e5f9f4fe5 +Merge: 2a9a2002e6a 5bfe2ba4690 +Author: Nhat Nguyen +Date: Fri Jun 1 23:09:29 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Avoid randomization bug in FeatureAwareTests + Adjust BWC version on client features + Add TRACE, CONNECT, and PATCH http methods (#31035) + Adjust BWC version on client features + [DOCS] Make geoshape docs less memory hungry (#31014) + Fix handling of percent-encoded spaces in Windows batch files (#31034) + [Docs] Fix a typo in Create Index naming limitation (#30891) + Introduce client feature tracking (#31020) + Ensure that index_prefixes settings cannot be changed (#30967) + REST high-level client: add delete ingest pipeline API (#30865) + [ML][TEST] Fix bucket count assertion in all tests in ModelPlotsIT (#31026) + Allow rollup job creation only if cluster is x-pack ready (#30963) + Fix interoperability with < 6.3 transport clients (#30971) + Add an option to split keyword field on whitespace at query time (#30691) + [Tests] Fix alias names in PutIndexTemplateRequestTests (#30960) + REST high-level client: add get ingest pipeline API (#30847) + Cross Cluster Search: preserve remote status code (#30976) + High-level client: list tasks failure to not lose nodeId (#31001) + [DOCS] Fixes links (#31011) + Watcher: Give test a little more time + Reuse expiration date of trial licenses (#30950) + Remove unused query methods from MappedFieldType. (#30987) + Transport client: Don't validate node in handshake (#30737) + [DOCS] Clarify not all PKCS12 usable as truststores (#30750) + HLRest: Allow caller to set per request options (#30490) + Remove version read/write logic in Verify Response (#30879) + [DOCS] Update readme for testing x-pack code snippets (#30696) + Ensure intended key is selected in SamlAuthenticatorTests (#30993) + Core: Remove RequestBuilder from Action (#30966) + +commit 5bfe2ba4690c2e55b207c583cff0d8bc98a07988 +Author: Jason Tedor +Date: Fri Jun 1 22:49:51 2018 -0400 + + Avoid randomization bug in FeatureAwareTests + + We compute a random version and later try to compute the version prior + that random version. If the random version is the earliest version in + our list of versions then it, by definition, does not have a previous + version. Yet trying to find its previous is someting we do and so the + test fails. This commit adds a version check to the randomization so + that we do not select the earliest version in our list. + +commit 3670a2ae059e82accae1f65bfd5d8767651e55ef +Author: Jason Tedor +Date: Fri Jun 1 17:11:34 2018 -0400 + + Adjust BWC version on client features + + This commit adjusts the BWC version on client features in master to + 6.3.0 after the functionality was backported to the 6.3 branch. + +commit f8785dda9df4a79fb0ff4cd23fb5f4fe98acff16 +Author: Tim Brooks +Date: Fri Jun 1 17:07:54 2018 -0600 + + Add TRACE, CONNECT, and PATCH http methods (#31035) + + This is related to #31017. That issue identified that these three http + methods were treated like GET requests. This commit adds them to + RestRequest. This means that these methods will be handled properly and + generate 405s. + +commit 2401150be75c1af06c5bcddca4a8d2ea7a4f084a +Author: Jason Tedor +Date: Fri Jun 1 16:32:14 2018 -0400 + + Adjust BWC version on client features + + This commit adjusts the BWC version on client features in master to + 6.4.0 after the functionality was backported to the 6.x branch. + +commit 7376c35960f8e08762d73aef53471678268d904b +Author: Igor Motov +Date: Fri Jun 1 15:05:37 2018 -0400 + + [DOCS] Make geoshape docs less memory hungry (#31014) + + Reduces shape size and precision in geo shape mapper examples to reduce + amount of memory required to check docs. + + Fixes #23836 + +commit 6fb1e4a75990c6ee4975604d146a8e1b44c78ab1 +Author: Jason Tedor +Date: Fri Jun 1 15:00:41 2018 -0400 + + Fix handling of percent-encoded spaces in Windows batch files (#31034) + + If you invoke elasticsearch-plugin (or any other CLI script on Windows) + with a path that has a percent-encoded space (or any other + percent-encoded character) because the CLI scripts now shell into a + common shell script (elasticsearch-cli) the percent-encoded space ends + up being interpreted as a parameter. For example passing install --batch + file:/c:/encoded%20%space/analysis-icu-7.0.0.zip to elasticsearch-plugin + leads to the %20 being interpreted as %2 followed by a zero. Here, the + %2 is interpreted as the second parameter (--batch) and the + InstallPluginCommand class ends up seeing + file:/c/encoded--batch0space/analysis-icu-7.0.0.zip as the path which + will not exist. This commit addresses this by escaping the %* that is + used to pass the parameters to the common CLI script so that the common + script sees the correct parameters without the %2 being substituted. + +commit 62da5dc17b0b0fd04dbe2f47688944e12c2f13bc +Author: Tal Levy +Date: Fri Jun 1 11:46:04 2018 -0700 + + add MoveToStepAction and RetryAction to the XPackClient Actions + +commit fbfb4ff8c75832dc9e111cdb6e8c1acfec1bca5e +Merge: 39844be808b a8c643e833d +Author: Tal Levy +Date: Fri Jun 1 11:44:38 2018 -0700 + + Merge branch 'master' into index-lifecycle + +commit 39844be808b737a6fb339b796ec70157dbd51c1a +Author: Colin Goodheart-Smithe +Date: Fri Jun 1 19:39:47 2018 +0100 + + Adds an explain API endpoint (#31005) + + * Adds an explain API endpoint + + This endpoint can be used to explain the current lifecycle state of an + index + + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/ExplainLifecycleAction.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/IndexExplainResponse.java + x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/inde + xlifecycle/action/TransportExplainLifecycleAction.java + x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/20_m + ove_to_step.yml + + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClien + tPlugin.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/ExplainLifecycleAction.java + x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/IndexExplainResponse.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifec + ycle/RandomStepInfo.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/ExplainLifecycleRequestTests.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/ExplainLifecycleResponseTests.java + x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifec + ycle/action/IndexExplainResponseTests.java + x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/inde + xlifecycle/IndexLifecycle.java + x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/inde + xlifecycle/action/RestExplainLifecycleAction.java + x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/inde + xlifecycle/action/TransportExplainLifecycleAction.java + x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/inde + xlifecycle/ExecuteStepsUpdateTaskTests.java + x-pack/plugin/index-lifecycle/src/test/java/org/elasticsearch/xpack/inde + xlifecycle/IndexLifecycleRunnerTests.java + x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.index_lifecycle + .explain_lifecycle.json + x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/20_m + ove_to_step.yml + x-pack/plugin/src/test/resources/rest-api-spec/test/index_lifecycle/30_e + xplain_lifecycle.yml + + * Adds tests for explain API + + * Addresses Review comments and fixes REST tests + + * Removes RequestBuilder from ExplainLifecycleAction + +commit a8c643e833d4e6064726b8747f63282a09594bc6 +Author: lipsill <39668292+lipsill@users.noreply.github.com> +Date: Fri Jun 1 18:15:24 2018 +0200 + + [Docs] Fix a typo in Create Index naming limitation (#30891) + +commit 4522b57e079a71c6f8f097dadae2405513f74e0f +Author: Jason Tedor +Date: Fri Jun 1 11:45:35 2018 -0400 + + Introduce client feature tracking (#31020) + + This commit introduces the ability for a client to communicate to the + server features that it can support and for these features to be used in + influencing the decisions that the server makes when communicating with + the client. To this end we carry the features from the client to the + underlying stream as we carry the version of the client today. This + enables us to enhance the logic where we make protocol decisions on the + basis of the version on the stream to also make protocol decisions on + the basis of the features on the stream. With such functionality, the + client can communicate to the server if it is a transport client, or if + it has, for example, X-Pack installed. This enables us to support + rolling upgrades from the OSS distribution to the default distribution + without breaking client connectivity as we can now elect to serialize + customs in the cluster state depending on whether or not the client + reports to us using the feature capabilities that it can under these + customs. This means that we would avoid sending a client pieces of the + cluster state that it can not understand. However, we want to take care + and always send the full cluster state during node-to-node communication + as otherwise we would end up with different understanding of what is in + the cluster state across nodes depending on which features they reported + to have. This is why when deciding whether or not to write out a custom + we always send the custom if the client is not a transport client and + otherwise do not send the custom if the client is transport client that + does not report to have the feature required by the custom. + + Co-authored-by: Yannick Welsch + +commit b8fda588f4c62c8703cf82cf0c6fd8bb18314fd1 +Author: Alan Woodward +Date: Fri Jun 1 15:17:35 2018 +0100 + + Ensure that index_prefixes settings cannot be changed (#30967) + +commit 11887fa54add1f5b2ee4f41e6a363f23c3296bc9 +Author: Sohaib Iftikhar +Date: Fri Jun 1 14:13:41 2018 +0200 + + REST high-level client: add delete ingest pipeline API (#30865) + + Relates to #27205 + +commit 0f12ecc95f6ec813edad2b9989ce28d9b45d75c3 +Author: Colin Goodheart-Smithe +Date: Fri Jun 1 11:40:59 2018 +0100 + + Fixes checkstyle error + x-pack/plugin/index-lifecycle/src/main/java/org/elasticsearch/xpack/inde + xlifecycle/LifecyclePolicySecurityClient.java + +commit 9141108334ddff880038fe935a7d6253bb8a63da +Author: Dimitris Athanasiou +Date: Fri Jun 1 10:51:12 2018 +0100 + + [ML][TEST] Fix bucket count assertion in all tests in ModelPlotsIT (#31026) + + This fixes the last remaining test that was missed in #30717. + + Closes #30715 + +commit e1649b8669f8c895c739bf4cd7077819711fd6c8 +Author: Yannick Welsch +Date: Fri Jun 1 10:47:53 2018 +0200 + + Allow rollup job creation only if cluster is x-pack ready (#30963) + + Otherwise we could end up with persistent tasks metadata in the cluster that some of the nodes + might not understand in case where the cluster is during rolling upgrade from the default 6.2 to the + default 6.3 distribution. + + Follow-up to #30743 + +commit fc98f09289d4e9962be8f6863d532eabd29a8b3a +Author: Colin Goodheart-Smithe +Date: Fri Jun 1 09:14:17 2018 +0100 + + Fixes compile errors caused by merge + + Specifically because `RequestBuilder`s have been removed from `Action`s + +commit 830198fa00ad616a65fd65c460f41389c15976a7 +Merge: 976935967b5 fb671adfd63 +Author: Colin Goodheart-Smithe +Date: Fri Jun 1 09:04:22 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit fb671adfd6364fc2efa942f19b17c7ce2fd0294c +Author: Yannick Welsch +Date: Fri Jun 1 10:02:57 2018 +0200 + + Fix interoperability with < 6.3 transport clients (#30971) + + With the default distribution changing in 6.3, clusters might now contain custom metadata that a + pure OSS transport client cannot deserialize. As this can break transport clients when accessing + the cluster state or reroute APIs, we've decided to exclude any custom metadata that the transport + client might not be able to deserialize. This will ensure compatibility between a < 6.3 transport + client and a 6.3 default distribution cluster. Note that this PR only covers interoperability with older + clients, another follow-up PR will cover full interoperability for >= 6.3 transport clients where we will + make it possible again to get the custom metadata from the cluster state. + + Relates to #30731 + +commit 0791f93dbd1c2e3626a0b7d2a513d1c444f9d493 +Author: Jim Ferenczi +Date: Fri Jun 1 09:47:03 2018 +0200 + + Add an option to split keyword field on whitespace at query time (#30691) + + This change adds an option named `split_queries_on_whitespace` to the `keyword` + field type. When set to true full text queries (`match`, `multi_match`, `query_string`, ...) that target the field will split the input on whitespace to build the query terms. Defaults to `false`. + Closes #30393 + +commit cea3c28b5b174e6356228b4c4eb6546f140041ba +Author: Christoph Büscher +Date: Fri Jun 1 09:45:04 2018 +0200 + + [Tests] Fix alias names in PutIndexTemplateRequestTests (#30960) + + The randomized alias names could contain unicode controll charactes that don't + survive an xContent rendering and parsing roundtrip when using the YAML xContent + type. This fix filters the randomized unicode string for control characters to + avoid this particular problem. + + Closes #30911 + +commit 80d20a90100d75efcd584f6d3826cf868c7654d0 +Author: Sohaib Iftikhar +Date: Fri Jun 1 08:55:43 2018 +0200 + + REST high-level client: add get ingest pipeline API (#30847) + + Relates to #27205 + +commit 70749e01c468664a5f88b5661ec5acb9ccb91557 +Author: Luca Cavanna +Date: Fri Jun 1 08:53:53 2018 +0200 + + Cross Cluster Search: preserve remote status code (#30976) + + In case an error is returned when calling search_shards on a remote + cluster, which will lead to throwing an exception in the coordinating + node, we should make sure that the status code returned by the + coordinating node is the same as the one returned by the remote + cluster. Up until now a 500 - Internal Server Error was always + returned. This commit changes this behaviour so that for instance if an + index is not found, which causes an 404, a 404 is also returned by the + coordinating node to the client. + + Closes #27461 + +commit 31351ab88036d8387f628030eb7f9c48070fef0f +Author: Luca Cavanna +Date: Fri Jun 1 08:53:24 2018 +0200 + + High-level client: list tasks failure to not lose nodeId (#31001) + + This commit reworks testing for `ListTasksResponse` so that random + fields insertion can be tested and xcontent equivalence can be checked + too. Proper exclusions need to be configured, and failures need to be + tested separately. This helped finding a little problem, whenever there + is a node failure returned, the nodeId was lost as it was never printed + out as part of the exception toXContent. + +commit 7c7431858001934ac989a552ff374ece8dbbd150 +Author: Lisa Cawley +Date: Thu May 31 17:02:18 2018 -0700 + + [DOCS] Fixes links (#31011) + +commit 2a9a2002e6aa21e48718ec74ea4d25a963964830 +Author: Nhat Nguyen +Date: Thu May 31 20:00:33 2018 -0400 + + CCR: Tighten requesting range check on leader + + This commit clarifies the origin of the global checkpoint that the + following shard uses and replaces illegal_state_exc E by an assertion. + + Relates #30980 + +commit ba78aa8c02b28b5a375fa2155d16e0d4b6aeaf0c +Merge: fa54be2dcd1 0f5e570184b +Author: Nhat Nguyen +Date: Thu May 31 19:56:30 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Deprecates indexing and querying a context completion field without context (#30712) + Refactor Sniffer and make it testable (#29638) + [Docs] Fix typo in Min Aggregation reference (#30899) + +commit 283d1bb2cf99379e31abfff517b29fd558325ec1 +Author: Nik Everett +Date: Thu May 31 17:47:48 2018 -0400 + + Watcher: Give test a little more time + + Changes watcher's integration tests to wait 30 seconds when starting + watcher rather than 10 seconds because this build failed when starting + took 12 seconds: + https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+6.3+periodic/222/console + +commit 3004b9eeec5858a45b505a2097ce04f198ca18fe +Author: Ioannis Kakavas +Date: Thu May 31 23:05:58 2018 +0300 + + Reuse expiration date of trial licenses (#30950) + + * Retain the expiryDate for trial licenses + + While updating the license signature to the new license spec retain + the trial license expiration date to that of the existing license. + + Resolves #30882 + +commit cd0a375414fd86bf2557dad1064c0b10e8312550 +Author: Julie Tibshirani +Date: Thu May 31 12:47:52 2018 -0700 + + Remove unused query methods from MappedFieldType. (#30987) + + * Remove MappedFieldType#nullValueQuery, as it is now unused. + * Remove MappedFieldType#queryStringTermQuery, as it is never overridden. + +commit fa54be2dcd1dfecdf065f5bb74066f97bcf138ac +Author: Nhat Nguyen +Date: Thu May 31 15:14:32 2018 -0400 + + CCR: Do not minimization requesting range on leader (#30980) + + Today before reading operations on the leading shard, we minimization + the requesting range with the global checkpoint. However, this might + make the request invalid if the following shard generates a requesting + range based on the global-checkpoint from a primary shard and sends + that request to a replica whose global checkpoint is lagged. + + Another issue is that we are mutating the request when applying + minimization. If the request becomes invalid on a replica, we will + reroute the mutated request instead of the original one to the primary. + + This commit removes the minimization and replaces it by a range check + with the local checkpoint. + +commit 4f66b9a27cf8070ed371de65609a0ff164879aa1 +Author: Tim Brooks +Date: Thu May 31 13:14:28 2018 -0600 + + Transport client: Don't validate node in handshake (#30737) + + This is related to #30141. Right now in the transport client we open a + temporary node connection and take the node information. This node + information is used to open a permanent connection that is used for the + client. However, we continue to use the configured transport address. + If the configured transport address is a load balancer, you might + connect to a different node for the permanent connection. This causes + the handshake validation to fail. This commit removes the handshake + validation for the transport client when it simple node sample mode. + +commit d9ab1469c9c15ef2cc25e7acd9d066315c784e6d +Author: Albert Zaharovits +Date: Thu May 31 21:49:27 2018 +0300 + + [DOCS] Clarify not all PKCS12 usable as truststores (#30750) + + Although elasticsearch-certutil generates PKCS#12 + files which are usable as both keystore and truststore + this is uncommon in practice. Settle these expectations + for the users following our security guides. + +commit b225f5e5c613160c3f6d16501add3ed1ae99e23d +Author: Nik Everett +Date: Thu May 31 13:59:52 2018 -0400 + + HLRest: Allow caller to set per request options (#30490) + + This modifies the high level rest client to allow calling code to + customize per request options for the bulk API. You do the actual + customization by passing a `RequestOptions` object to the API call + which is set on the `Request` that is generated by the high level + client. It also makes the `RequestOptions` a thing in the low level + rest client. For now that just means you use it to customize the + headers and the `httpAsyncResponseConsumerFactory` and we'll add + node selectors and per request timeouts in a follow up. + + I only implemented this on the bulk API because it is the first one + in the list alphabetically and I wanted to keep the change small + enough to review. I'll convert the remaining APIs in a followup. + +commit d826cb36c3111f67d29e750620e7727a238b90da +Author: Michael Basnight +Date: Thu May 31 12:10:01 2018 -0500 + + Remove version read/write logic in Verify Response (#30879) + + Since master will always communicate with a >=6.4 node, the logic for + checking if the node is 6.4 and conditionally reading and writing based + on that can be removed from master. This logic will stay in 6.x as it is + the bridge to the cleaner response in master. This also unmutes the + failing test due to this bwc change. + + Closes #30807 + +commit 3960a7a310a9591cf61caea99af0d098be9a0317 +Author: Lisa Cawley +Date: Thu May 31 09:32:22 2018 -0700 + + [DOCS] Update readme for testing x-pack code snippets (#30696) + +commit 532641ef30b0b095e8207563510e1454d5dca87d +Author: Ioannis Kakavas +Date: Thu May 31 18:09:32 2018 +0300 + + Ensure intended key is selected in SamlAuthenticatorTests (#30993) + + * Ensure that a purposefully wrong key is used + + Uses a specific keypair for tests that require a purposefully wrong + keypair instead of selecting one randomly from the same pull from + which the correct one is selected. Entropy is low because of the + small space and the same key can be randomly selected as both the + correct one and the wrong one, causing the tests to fail. + The purposefully wrong key is also used in + testSigningKeyIsReloadedForEachRequest and needs to be cleaned + up afterwards so the rest of the tests don't use that for signing. + + Resolves #30970 + +commit 46e8d97813addd8c57fa54d2c700d26a171f2dbb +Author: Ryan Ernst +Date: Thu May 31 16:15:00 2018 +0200 + + Core: Remove RequestBuilder from Action (#30966) + + This commit removes the RequestBuilder generic type from Action. It was + needed to be used by the newRequest method, which in turn was used by + client.prepareExecute. Both of these methods are now removed, along with + the existing users of prepareExecute constructing the appropriate + builder directly. + +commit 0f5e570184b42ed50c658a391e78956f0f4b4352 +Author: Jim Ferenczi +Date: Thu May 31 16:09:48 2018 +0200 + + Deprecates indexing and querying a context completion field without context (#30712) + + This change deprecates completion queries and documents without context that target a + context enabled completion field. Querying without context degrades the search + performance considerably (even when the number of indexed contexts is low). + This commit targets master but the deprecation will take place in 6.x and the functionality + will be removed in 7 in a follow up. + + Closes #29222 + +commit 63f3a6113462525f8c882b0cbb7e364509f9566e +Author: Luca Cavanna +Date: Thu May 31 16:05:09 2018 +0200 + + Refactor Sniffer and make it testable (#29638) + + This commit reworks the Sniffer component to simplify it and make it possible to test it. + + In particular, it no longer takes out the host that failed when sniffing on failure, but rather relies on whatever the cluster returns. This is the result of some valid comments from #27985. Taking out one single host is too naive, hard to test and debug. + + A new Scheduler abstraction is introduced to abstract the tasks scheduling away and make it possible to plug in any test implementation and take out timing aspects when testing. + + Concurrency aspects have also been improved, synchronized methods are no longer required. At the same time, we were able to take #27697 and #25701 into account and fix them, especially now that we can more easily add tests. + + Last but not least, unit tests are added for the Sniffer component, long overdue. + + Closes #27697 + Closes #25701 + +commit 976935967b5dc28e09279aecb58cc8045c4d9a64 +Merge: 2f2832db041 4777d8a2df0 +Author: Colin Goodheart-Smithe +Date: Thu May 31 14:29:00 2018 +0100 + + Merge branch 'master' into index-lifecycle + +commit 7e8cf768cf53d9cb7da7e8c189c92d2a3adccdf8 +Author: Martijn van Groningen +Date: Thu May 31 15:15:48 2018 +0200 + + changed persistent task name to be of similar structure as the others + +commit 4777d8a2df041dd69e3cd727255068914a4b8f3e +Author: Christoph Büscher +Date: Thu May 31 15:05:03 2018 +0200 + + [Docs] Fix typo in Min Aggregation reference (#30899) + +commit 0176100a73295708c580bb3376ab6f169ccfb543 +Merge: 44b6dcd2a59 0ff2c605b6f +Author: Nhat Nguyen +Date: Thu May 31 07:42:30 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [CI] Mute Ml rolling upgrade test for mixed cluster too + +commit 0ff2c605b6ff3da59fbaa2f6ac3407fd52249424 +Author: David Roberts +Date: Thu May 31 11:17:18 2018 +0100 + + [CI] Mute Ml rolling upgrade test for mixed cluster too + + It can fail in either the mixed cluster or the upgraded cluster, + so it needs to be muted in both. + + Tracked by #30982 + +commit 44b6dcd2a593a5a76656edbc36d7277c22151ad5 +Merge: a82f2e31b4b c41574376f0 +Author: Tanguy Leroux +Date: Thu May 31 12:04:21 2018 +0200 + + Merge branch 'master' into ccr + + * master: + Make AllocatedPersistentTask.isCompleted() protected (#30949) + +commit c41574376f0295051b44b41bf627b76104ea6d4a +Author: Tanguy Leroux +Date: Thu May 31 09:19:05 2018 +0200 + + Make AllocatedPersistentTask.isCompleted() protected (#30949) + + This commit changes the isCompleted() method to be protected so that + classes that extends AllocatedPersistentTask can use it. + + Related to #30858 + +commit a82f2e31b4b00cc25b07afb59ed2295a5279aa94 +Author: Martijn van Groningen +Date: Thu May 31 08:03:53 2018 +0200 + + [CCR] Also copy routing_num_shards from leader to follow index. (#30894) + + Bug was introduced when create and follow api was added in #30602 + +commit 16402305bbc77026f9ea49ef178f7584cf21f334 +Merge: 8793ebcda09 8e4ab82e3df +Author: Nhat Nguyen +Date: Wed May 30 21:36:56 2018 -0400 + + Merge branch 'master' into ccr + + * master: + [CI] Mute Ml rolling upgrade tests + Fix license on AcitveDirectorySIDUtil (#30972) + [Test] Prefer ArrayList over Vector (#30965) + [CI] Mute HttpSecretsIntegrationTests#testWebhookAction test + Mute FlushIT tests + Add “took” timing info to response for _msearch/template API (#30961) + +commit 8e4ab82e3df0024274a44a9c3dcdd3eb153bb92c +Author: Igor Motov +Date: Wed May 30 21:30:18 2018 -0400 + + [CI] Mute Ml rolling upgrade tests + + Tracked by #30982 + +commit 8793ebcda093f66d69e46fc6c19023686b0539da +Author: Nhat Nguyen +Date: Wed May 30 18:36:06 2018 -0400 + + Limit num hits when reading Lucene changes (#30908) + + Today we don't limit the number of hits when reading changes from Lucene + index. If the index and the requesting seq# range both are large, the + searcher may consume a huge amount of memory. + + This commit uses a fixed size batch with search_after to avoid the + problem. + +commit 735f232e0340e674ed1ac5ccab81976760179ac9 +Author: Jason Tedor +Date: Wed May 30 15:28:25 2018 -0400 + + Fix license on AcitveDirectorySIDUtil (#30972) + + This code is from an Apache 2.0 licensed codebase and when we imported + it into our codebase it carried the Apache 2.0 license as well. However, + during the migration of the X-Pack codebase from the internal private + repository to the elastic/elasticsearch repository, the migration tool + mistakently changed the license on this source file from the Apache 2.0 + license to the Elastic license. This commit addresses this mistake by + reapplying the Apache 2.0 license. + +commit 0a5d46ef3cd05169e45b61e9032c03883e9eaef6 +Author: Christoph Büscher +Date: Wed May 30 21:11:49 2018 +0200 + + [Test] Prefer ArrayList over Vector (#30965) + + Replaces some occurances of Vector class with ArrayList in + tests of the rank-eval module. + +commit f25ee254cc5ce8b4fd58ebce9eb55dbf48c48b72 +Author: Nhat Nguyen +Date: Wed May 30 14:29:58 2018 -0400 + + Mute ShardChangesIT#testFollowIndex + +commit 1da26a42cb1ffd4c9bf027a5d0959ebbfd365bc9 +Author: Igor Motov +Date: Wed May 30 14:29:06 2018 -0400 + + [CI] Mute HttpSecretsIntegrationTests#testWebhookAction test + + Tracked by #30094 + +commit b834254862214a0a69b42256455c243d3e2d9945 +Author: Nhat Nguyen +Date: Wed May 30 14:23:53 2018 -0400 + + Mute FlushIT tests + + We have identified the source causing these tests failed. + This commit mutes them again until we have a proper fix. + + Relates #29392 + +commit facbb2b2dceb8a3c50a3147adab676c39c10c74e +Author: markharwood +Date: Wed May 30 17:57:28 2018 +0100 + + Add “took” timing info to response for _msearch/template API (#30961) + + Add “took” timing info to response for _msearch/template API + Closes #30957 + +commit 5e81a201710218d2ccccc37d7468f13fb6bd02b8 +Merge: adca32eae74 b716b081971 +Author: Nhat Nguyen +Date: Wed May 30 12:11:52 2018 -0400 + + Merge branch 'master' into ccr + + * master: + Add Verify Repository High Level REST API (#30934) + [CI] Mute SamlAuthenticatorTests testIncorrectSigningKeyIsRejected + [DOCS] Fixes kibana security file location + SQL: Remove log4j and joda from JDBC dependencies (#30938) + Revert accidentally pushed changes in NoriAnalysisTests + Fix composite agg serialization error + Change ScriptException status to 400 (bad request) (#30861) + Fix synced flush docs + REST high-level client: add synced flush API (2) (#30650) + Fix missing option serialization after backport + Cross Cluster Search: do not use dedicated masters as gateways (#30926) + Fix AliasMetaData parsing (#30866) + Fsync state file before exposing it (#30929) + +commit b716b0819717076518dad7ee9a15716dcdb50347 +Author: Michael Basnight +Date: Wed May 30 11:10:00 2018 -0500 + + Add Verify Repository High Level REST API (#30934) + + This commit adds Verify Repository, the associated docs and tests for + the high level REST API client. A few small changes to the Verify + Repository Response went into the commit as well. + + Relates #27205 + +commit 96b4dae221057b4ea0c6f453b84376dd510d95d1 +Author: Igor Motov +Date: Wed May 30 11:29:17 2018 -0400 + + [CI] Mute SamlAuthenticatorTests testIncorrectSigningKeyIsRejected + + Tracked by #30970 + +commit 94e1bdba331362dfb05c73b6e086566ecef35cc4 +Author: lcawl +Date: Wed May 30 07:47:20 2018 -0700 + + [DOCS] Fixes kibana security file location + +commit cdbbbafb9f8b2f3fe80e73bc71efccff65800a35 +Author: Igor Motov +Date: Wed May 30 09:57:32 2018 -0400 + + SQL: Remove log4j and joda from JDBC dependencies (#30938) + + More cleanup of JDBC driver project + + Relates to #29856 + +commit e852eb0548094f042ccf7ab585045a7176fd3e93 +Author: Jim Ferenczi +Date: Wed May 30 15:34:55 2018 +0200 + + Revert accidentally pushed changes in NoriAnalysisTests + +commit 532b91ffa676986c013c12a43cc87c5aae5d68bc +Author: Jim Ferenczi +Date: Wed May 30 14:22:48 2018 +0200 + + Fix composite agg serialization error + + Fix serialization after backport + + Relates #29465 + +commit 1ea9f11b035f45e0d51be8cf64126e4a32e62ae7 +Author: Christoph Büscher +Date: Wed May 30 14:00:07 2018 +0200 + + Change ScriptException status to 400 (bad request) (#30861) + + Currently failures to compile a script usually lead to a ScriptException, which + inherits the 500 INTERNAL_SERVER_ERROR from ElasticsearchException if it does + not contain another root cause. Instead, this should be a 400 Bad Request error. + This PR changes this more generally for script compilation errors by changing + ScriptException to return 400 (bad request) as status code. + + Closes #12315 + +commit 7c5abc0a6b32899490d6566363df03ab72cd8915 +Author: Nik Everett +Date: Wed May 30 07:50:44 2018 -0400 + + Fix synced flush docs + + They had some copy and paste errors that failed the docs build. + +commit 2c27c58718f995793883a002e5e9ddcaae124cb7 +Author: Sohaib Iftikhar +Date: Wed May 30 13:32:52 2018 +0200 + + REST high-level client: add synced flush API (2) (#30650) + + Adds the synced flush API to the high level REST client. + + Relates to #27205. + +commit f582418ada24f34edd21c4c98b31b7b55df41f6b +Author: Jim Ferenczi +Date: Wed May 30 12:55:31 2018 +0200 + + Fix missing option serialization after backport + + Relates #29465 + +commit adca32eae748a1423f27885689c2dd013d908af8 +Author: Martijn van Groningen +Date: Wed May 30 12:42:01 2018 +0200 + + no need to resolve index name as only concrete index names are used + +commit 3c21e46fa3b651c2b56379d2687799649b6138e4 +Author: Luca Cavanna +Date: Wed May 30 12:32:41 2018 +0200 + + Cross Cluster Search: do not use dedicated masters as gateways (#30926) + + When we are connecting to a remote cluster we should never select + dedicated master nodes as gateway nodes, or we will end up loading them + with requests that should rather go to other type of nodes e.g. data + nodes or coord_only nodes. + + This commit adds the selection based on the node role, to the existing + selection based on version and potential node attributes. + + Closes #30687 + +commit 6341d101d2d8b748eaf698aebaf01d4d78d4a227 +Author: olcbean <26058559+olcbean@users.noreply.github.com> +Date: Wed May 30 12:31:24 2018 +0200 + + Fix AliasMetaData parsing (#30866) + + AliasMetaData should be parsed more leniently so that the high-level REST client can support forward compatibility on it. This commit addresses this issue that was found as part of #28799 and adds dedicated XContent tests as well. + +commit 4a20dca5fee943d63954362fd023f171b6fab3d4 +Author: Martijn van Groningen +Date: Wed May 30 10:26:49 2018 +0200 + + Required changes after merging in master. + +commit ff8ce2c575ec579710694528609d5dcfe2356fa0 +Author: Yannick Welsch +Date: Wed May 30 10:15:12 2018 +0200 + + Fsync state file before exposing it (#30929) + + With multiple data paths, we write the state files for index metadata to all data paths. We only properly fsync on the first location, though. For other locations, we possibly expose the file before its contents is properly fsynced. This can lead to situations where, after a crash, and where the first data path is not available anymore, ES will see a partially-written state file, preventing the node to start up. + +commit 56472d650513af19bbc7fad172923145606f50dc +Merge: 03e3bd28c9b e33d107f840 +Author: Martijn van Groningen +Date: Wed May 30 09:55:34 2018 +0200 + + Merge remote-tracking branch 'es/master' into ccr + + * es/master: (24 commits) + Add missing_bucket option in the composite agg (#29465) + Rename index_prefix to index_prefixes (#30932) + Rename methods in PersistentTasksService (#30837) + [DOCS] Fix watcher file location + Update the version checks around range bucket keys, now that the change was backported. + Use dedicated ML APIs in tests (#30941) + [DOCS] Remove reference to platinum Docker image (#30916) + Minor clean-up in InternalRange. (#30886) + stable filemode for zip distributions (#30854) + [DOCS] Adds missing TLS settings for auditing (#30822) + [test] packaging: use shell when running commands (#30852) + Fix location of AbstractHttpServerTransport (#30888) + [test] packaging test logging for suse distros + Moved keyword tokenizer to analysis-common module (#30642) + Upgrade to Lucene-7.4-snapshot-1cbadda4d3 (#30928) + Limit the scope of BouncyCastle dependency (#30358) + [DOCS] Reset edit links (#30909) + Fix IndexTemplateMetaData parsing from xContent (#30917) + Remove log traces in AzureStorageServiceImpl and fix test (#30924) + Deprecate accepting malformed requests in stored script API (#28939) + ... + +commit e33d107f840602344774d1b71774d04f834c74e0 +Author: Jim Ferenczi +Date: Wed May 30 09:48:40 2018 +0200 + + Add missing_bucket option in the composite agg (#29465) + + This change adds a new option to the composite aggregation named `missing_bucket`. + This option can be set by source and dictates whether documents without a value for the + source should be ignored. When set to true, documents without a value for a field emits + an explicit `null` value which is then added in the composite bucket. + The `missing` option that allows to set an explicit value (instead of `null`) is deprecated in this change and will be removed in a follow up (only in 7.x). + This commit also changes how the big arrays are allocated, instead of reserving + the provided `size` for all sources they are created with a small intial size and they grow + depending on the number of buckets created by the aggregation: + Closes #29380 + +commit 67905c85a5331e62bfd0d4685e66f39295567022 +Author: Alan Woodward +Date: Wed May 30 08:32:31 2018 +0100 + + Rename index_prefix to index_prefixes (#30932) + + This commit also adds index_prefixes tests to TextFieldMapperTests to ensure that cloning and wire-serialization work correctly + +commit a0af0e7f1e973e11ac11711740cbac78f3f18f88 +Author: Tanguy Leroux +Date: Wed May 30 09:20:14 2018 +0200 + + Rename methods in PersistentTasksService (#30837) + + This commit renames methods in the PersistentTasksService, to + make obvious that the methods send requests in order to change + the state of persistent tasks. + + Relates to #29608. + +commit b5c997b10bd3d5341911a0b02b7a29da71b55fc2 +Author: lcawl +Date: Tue May 29 21:55:56 2018 -0700 + + [DOCS] Fix watcher file location + +commit 913778b37a076e963b84df483f5c77e72f62c6c6 +Author: Julie Tibshirani +Date: Tue May 29 20:51:27 2018 -0700 + + Update the version checks around range bucket keys, now that the change was backported. + +commit 2f2832db041fd454ebfe1e47abf15dd914d38a2a +Author: Tal Levy +Date: Tue May 29 19:16:00 2018 -0700 + + add _retry API to index lifecycle policies (#30769) + +commit bcfdccaf3f399e93d6883fe6363e375aba30ada6 +Author: Jason Tedor +Date: Tue May 29 21:17:47 2018 -0400 + + Use dedicated ML APIs in tests (#30941) + + ML has dedicated APIs for datafeeds and jobs yet base test classes and + some tests were relying on the cluster state for this state. This commit + removes this usage in favor of using the dedicated endpoints. + +commit 83a7ade7c5e34bdd6985dec10bbfdc40a31c40c4 +Author: Toby McLaughlin +Date: Wed May 30 11:17:02 2018 +1000 + + [DOCS] Remove reference to platinum Docker image (#30916) + +commit a79c5bd3d65a5f26b69931f7323b81d41d22716b +Author: Julie Tibshirani +Date: Tue May 29 18:16:02 2018 -0700 + + Minor clean-up in InternalRange. (#30886) + + * Make sure all instance variables are final. + * Make generateKey a private static method, instead of protected. + * Rename formatter -> format for consistency. + * Serialize bucket keys as strings as opposed to optional strings. + * Pull the stream serialization logic for buckets into the Bucket class. + +commit 116d08303e73b250d854ad0f66d37fe7e98a4d62 +Author: Andy Bristol +Date: Tue May 29 17:32:06 2018 -0700 + + stable filemode for zip distributions (#30854) + + Applies default file and directory permissions to zip distributions + similar to how they're set for the tar distributions. Previously zip + distributions would retain permissions they had on the build host's + working tree, which could vary depending on its umask + + For #30799 + +commit eb2e43b6955594ba4e1abcf6726dafbf2cbe1227 +Author: Lisa Cawley +Date: Tue May 29 15:53:19 2018 -0700 + + [DOCS] Adds missing TLS settings for auditing (#30822) + +commit 4001097a687693ef0783689b50c3220a2be02ee7 +Author: Andy Bristol +Date: Tue May 29 15:17:11 2018 -0700 + + [test] packaging: use shell when running commands (#30852) + + When subprocesses are started with ProcessBuilder, they're forked by the + java process directly rather than from a shell, which can be surprising + for our use case here in the packaging tests which is similar to + scripting. + + This commit changes the tests to run their subprocess commands in a + shell, using the bash -c